diff --git a/.gitignore b/.gitignore index ca133bae1..fe9b9237f 100644 --- a/.gitignore +++ b/.gitignore @@ -231,40 +231,99 @@ urbackup/server_ident_ecdsa409k1.priv urbackup/server_ident_ecdsa409k1.pub clientctl/Debug/* urbackup/new_version_deb/* + +urbackupserver/www/js/templates.js.new +urbackupserver/www/templates.js.new +clientctl/Release/* +urbackup/UrBackupUpdate.sig2 +backup_scripts/* +.vs/UrBackupBackend/v14/.suo +clientctl/x64/* +UrBackupBackend.VC.opendb +deps/ +urbackup/backup_server_*.db* +urbackup_ecdsa409k1.pub +urbackup/UrBackupUpdateLinux.sh +urbackup/UrBackupUpdateLinux.sig2 +urbackup/UrBackupUpdateMac.sh +urbackup/UrBackupUpdateMac.sig2 +urbackupserver/www/templates/node_modules/.bin/dustc +SQLGen/SQLGen.VC.opendb +SQLGen/.vs/* +SQLGen/SQLGen.VC.db +SQLGen/SQLGen.VC.VC.opendb +UrBackupBackend.VC.db +UrBackupBackend.VC.VC.opendb +urbackup/version_linux.txt +urbackup/version_osx.txt +urbackup/hdat_file_*_.dat +urbackup/windows_components_config/* +SQLGen/x64/* +luaplugin/x64/* +blockalign/x64/* +/.vs/* +/blockalign_src/Debug/* +/blockalign_src/x64/* +/build/__pycache__/* +/blockalign/Debug/* +/luaplugin/Debug/* +/urbackupserver/www/templates/post/* +/vcpkg_installed +/urbackupserver/Release +/blockalign_src/Release +/luaplugin/Release +*.o +autom4te.cache +.deps +.dirstamp +cryptoplugin/src/m4/lt~obsolete.m4 +cryptoplugin/src/m4/ltoptions.m4 +cryptoplugin/src/m4/ltsugar.m4 +cryptoplugin/src/m4/ltversion.m4 +aclocal.m4 +blockalign +compile +config.guess +config.h +config.h.in +config.log +config.status +config.sub +configure +configure.ac +curr_build +defaults +depcomp +init.d +install-sh +Makefile +Makefile.am +Makefile.in +missing +stamp-h1 +urbackupclient_dmsnaptool +urbackupclientbackend +urbackupclientctl +cryptoplugin/src/local.h.in +cryptoplugin/src/ltmain.sh +cryptoplugin/src/Makefile.in +cryptoplugin/src/missing +cryptoplugin/src/m4/libtool.m4 -urbackupserver/www/js/templates.js.new -urbackupserver/www/templates.js.new -clientctl/Release/* -urbackup/UrBackupUpdate.sig2 -backup_scripts/* -.vs/UrBackupBackend/v14/.suo -clientctl/x64/* -UrBackupBackend.VC.opendb -deps/ -urbackup/backup_server_*.db* -urbackup_ecdsa409k1.pub -urbackup/UrBackupUpdateLinux.sh -urbackup/UrBackupUpdateLinux.sig2 -urbackup/UrBackupUpdateMac.sh -urbackup/UrBackupUpdateMac.sig2 -urbackupserver/www/templates/node_modules/.bin/dustc -SQLGen/SQLGen.VC.opendb -SQLGen/.vs/* -SQLGen/SQLGen.VC.db -SQLGen/SQLGen.VC.VC.opendb -UrBackupBackend.VC.db -UrBackupBackend.VC.VC.opendb -urbackup/version_linux.txt -urbackup/version_osx.txt -urbackup/hdat_file_*_.dat -urbackup/windows_components_config/* -SQLGen/x64/* -luaplugin/x64/* -blockalign/x64/* -/.vs/* -/blockalign_src/Debug/* -/blockalign_src/x64/* -/build/__pycache__/* -/blockalign/Debug/* -/luaplugin/Debug/* -/urbackupserver/www/templates/post/* +/urbackupclient/restorewww/node_modules +/btrfs/btrfsplugin/x64 +/btrfs/fuse/oslib/x64 +/btrfs/fuse/x64 +/ARM64/* +/blockalign_src/ARM64 +/clientctl/ARM64 +/cryptoplugin/ARM64 +/fileservplugin/ARM64 +/fsimageplugin/ARM64 +/httpserver/ARM64 +/luaplugin/ARM64 +/urbackupclient/ARM64 +/urbackupclient/sysvol_test/ARM64 +/urbackupserver/ARM64 +/urlplugin/ARM64 +/SQLGen/SQLGen/x64 diff --git a/CompiledServer.vcxproj b/CompiledServer.vcxproj index c9a2bc50d..b65ea1a57 100644 --- a/CompiledServer.vcxproj +++ b/CompiledServer.vcxproj @@ -1,6 +1,10 @@  + + Debug + ARM64 + Debug Win32 @@ -9,6 +13,10 @@ Debug x64 + + Release Service + ARM64 + Release Service Win32 @@ -17,6 +25,10 @@ Release Service x64 + + Release + ARM64 + Release Win32 @@ -38,35 +50,52 @@ Application Unicode true - v142 + v143 Application Unicode true - v142 + v143 Application Unicode - v142 + v143 Application Unicode true - v142 + v143 + + + Application + Unicode + true + v143 Application Unicode true - v142 + v143 + + + Application + Unicode + true + v143 Application Unicode - v142 + v143 + + + Application + Unicode + v143 @@ -83,12 +112,21 @@ + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 @@ -96,38 +134,83 @@ $(Configuration)\ true $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ $(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ true + true $(SolutionDir)$(Configuration)\ $(Configuration)\ false $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ $(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ false + false $(SolutionDir)$(Configuration)\ $(Configuration)\ false $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ $(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ false + false AllRules.ruleset AllRules.ruleset + AllRules.ruleset + + AllRules.ruleset AllRules.ruleset + AllRules.ruleset + + AllRules.ruleset AllRules.ruleset + AllRules.ruleset + + + + + true + + + x64-windows-static-md + + + arm64-windows-static-md + + + x64-windows-static-md + + + x64-windows-static-md + + + x86-windows-static-md + + + x64-windows-static-md + + + x64-windows-static-md + + + x86-windows-static-md @@ -172,16 +255,41 @@ 4005;%(DisableSpecificWarnings) - ./libx64;D:\Developement\urbackup_libs\libx64;%(AdditionalLibraryDirectories) + %(AdditionalLibraryDirectories) true Console - false + true MachineX64 Dbghelp.lib;%(AdditionalDependencies) + + + + Disabled + ./libfastcgi;%(AdditionalIncludeDirectories) + DO_NOT_USE_CRYPTOPP_MD5;WIN32;_DEBUG;_CONSOLE;SQLITE_ENABLE_UNLOCK_NOTIFY;SQLITE_ENABLE_DBPAGE_VTAB;THREAD_BOOST;%(PreprocessorDefinitions) + true + EnableFastChecks + MultiThreadedDebugDLL + + + Level3 + ProgramDatabase + 4005;%(DisableSpecificWarnings) + + + %(AdditionalLibraryDirectories) + true + Console + true + + + Dbghelp.lib;%(AdditionalDependencies) + + ./libfastcgi_win;%(AdditionalIncludeDirectories) @@ -193,7 +301,7 @@ ProgramDatabase - libx86;D:\Developement\urbackup_libs\libx86;%(AdditionalLibraryDirectories) + %(AdditionalLibraryDirectories) true Console true @@ -223,11 +331,36 @@ Console true true - false + true MachineX64 - libx64\;D:\Developement\urbackup_libs\libx64 + + + Dbghelp.lib;%(AdditionalDependencies) + + + + + + ./libfastcgi_win;%(AdditionalIncludeDirectories) + WIN32;NDEBUG;_CONSOLE;SQLITE_ENABLE_UNLOCK_NOTIFY;SQLITE_ENABLE_DBPAGE_VTAB;THREAD_BOOST;DO_NOT_USE_CRYPTOPP_MD5;%(PreprocessorDefinitions) + MultiThreadedDLL + + + Level3 + ProgramDatabase + + + true + Console + true + true + true + + + + Dbghelp.lib;%(AdditionalDependencies) @@ -273,13 +406,36 @@ Console true true - false + true MachineX64 Dbghelp.lib;%(AdditionalDependencies) + + + + ./libfastcgi_win;%(AdditionalIncludeDirectories) + WIN32;NDEBUG;_CONSOLE;AS_SERVICE;SQLITE_ENABLE_UNLOCK_NOTIFY;SQLITE_ENABLE_DBPAGE_VTAB;DO_NOT_USE_CRYPTOPP_MD5;%(PreprocessorDefinitions) + MultiThreadedDLL + + + Level3 + ProgramDatabase + + + libx64/;D:\Developement\urbackup_libs\libx64;%(AdditionalLibraryDirectories) + true + Console + true + true + true + + + Dbghelp.lib;%(AdditionalDependencies) + + diff --git a/Database.cpp b/Database.cpp index fee67f91c..80774f6ff 100644 --- a/Database.cpp +++ b/Database.cpp @@ -1,777 +1,822 @@ -/************************************************************************* -* UrBackup - Client/Server backup system -* Copyright (C) 2011-2016 Martin Raiber -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -**************************************************************************/ -#ifndef NO_SQLITE - -#if defined(_WIN32) || defined(WIN32) -#define _CRT_SECURE_NO_WARNINGS -#endif - -#include "vld.h" -#ifndef BDBPLUGIN -#include "Server.h" -#else -#ifdef LINUX -#include "bdbplugin/config.h" -#include DB_HEADER -#else -#include -#endif -#include "Interface/Server.h" -#endif -#include "Query.h" -#ifdef USE_SYSTEM_SQLITE -#include -#else -#include "sqlite/sqlite3.h" -#endif -#include "Interface/File.h" -#include -extern "C" -{ - #include "sqlite/shell.h" -} -#include "Database.h" -#include "stringtools.h" - - -namespace -{ - size_t get_sqlite_cache_size() - { - std::string cache_size_str = Server->getServerParameter("sqlite_cache_size"); - - if(!cache_size_str.empty()) - { - return atoi(cache_size_str.c_str()); - } - else - { - return 2*1024; //2MB - } - } - - void errorLogCallback(void *pArg, int iErrCode, const char *zMsg) - { - switch (iErrCode) - { - case SQLITE_LOCKED: - case SQLITE_BUSY: - case SQLITE_SCHEMA: - return; - case SQLITE_NOTICE_RECOVER_ROLLBACK: - case SQLITE_NOTICE_RECOVER_WAL: - Server->Log("SQLite: "+ std::string(zMsg) + " code: " + convert(iErrCode), LL_INFO); - break; - default: - Server->Log("SQLite: " + std::string(zMsg) + " errorcode: " + convert(iErrCode), LL_WARNING); - break; - } - } -} - - -struct UnlockNotification { - bool fired; - ICondition* cond; - IMutex *mutex; -}; - -static void unlock_notify_cb(void **apArg, int nArg) -{ - for(int i=0; imutex); - p->fired = true; - p->cond->notify_all(); - } -} - -CDatabase::~CDatabase() -{ - destroyAllQueries(); - for(std::map::iterator iter=prepared_queries.begin();iter!=prepared_queries.end();++iter) - { - CQuery *q=(CQuery*)iter->second; - delete q; - } - prepared_queries.clear(); - - sqlite3_close(db); -} - -bool CDatabase::Open(std::string pFile, const std::vector > &attach, - size_t allocation_chunk_size, ISharedMutex* p_single_user_mutex, IMutex* p_lock_mutex, - int* p_lock_count, ICondition *p_unlock_cond, const str_map& p_params) -{ - single_user_mutex = p_single_user_mutex; - lock_mutex = p_lock_mutex; - lock_count = p_lock_count; - unlock_cond = p_unlock_cond; - params = p_params; - - attached_dbs=attach; - in_transaction=false; - if( sqlite3_open(pFile.c_str(), &db) ) - { - Server->Log("Could not open db ["+pFile+"]"); - sqlite3_close(db); - db = NULL; - return false; - } - else - { - str_map::const_iterator it = params.find("synchronous"); - if (it != params.end()) - { - Write("PRAGMA synchronous="+it->second); - } - else - { - Write("PRAGMA synchronous=NORMAL"); - } - Write("PRAGMA foreign_keys = ON"); - Write("PRAGMA threads = 2"); - - it = params.find("wal_autocheckpoint"); - if (it != params.end()) - { - Write("PRAGMA wal_autocheckpoint=" + it->second); - - if (watoi(it->second)<=0) - { - int enable = 1; - sqlite3_file_control(db, NULL, SQLITE_FCNTL_PERSIST_WAL, &enable); -#ifdef SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE - int was_enabled = 0; - sqlite3_db_config(db, SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, 1, &was_enabled); -#endif - } - } - - it = params.find("page_size"); - if (it != params.end()) - { - Write("PRAGMA page_size=" + it->second); - } - else - { - Write("PRAGMA page_size=4096"); - } - - it = params.find("mmap_size"); - if (it != params.end()) - { - Write("PRAGMA mmap_size=" + it->second); - } - - if(allocation_chunk_size!=std::string::npos) - { - int chunk_size = static_cast(allocation_chunk_size); - sqlite3_file_control(db, NULL, SQLITE_FCNTL_CHUNK_SIZE, &chunk_size); - } - - static size_t sqlite_cache_size = get_sqlite_cache_size(); - Write("PRAGMA cache_size = -"+convert(sqlite_cache_size)); - - sqlite3_busy_timeout(db, c_sqlite_busy_timeout_default); - -#if defined(_DEBUG) || (!defined(_WIN32) && !defined(NDEBUG)) - if (Server->getRandomNumber() % 2 == 0) - { - Write("PRAGMA reverse_unordered_selects = ON"); - } -#endif - - AttachDBs(); - - return true; - } -} - -void CDatabase::initMutex(void) -{ - sqlite3_config(SQLITE_CONFIG_LOG, errorLogCallback, NULL); -} - -void CDatabase::destroyMutex(void) -{ -} - -db_results CDatabase::Read(std::string pQuery) -{ - //Server->Log("SQL Query(Read): "+pQuery, LL_DEBUG); - IQuery *q=Prepare(pQuery, false); - if(q!=NULL) - { - db_results ret=q->Read(); - delete ((CQuery*)q); - return ret; - } - return db_results(); -} - -bool CDatabase::Write(std::string pQuery) -{ - //Server->Log("SQL Query(Write): "+pQuery, LL_DEBUG); - IQuery *q=Prepare(pQuery, false); - if(q!=NULL) - { - bool b=q->Write(); - delete ((CQuery*)q); - return b; - } - else - { - return false; - } -} - -//ToDo: Cache Writings - -bool CDatabase::BeginReadTransaction() -{ - if (write_lock.get() == NULL) - { - transaction_read_lock.reset(new IScopedReadLock(single_user_mutex)); - } - - in_transaction = true; - if(Write("BEGIN")) - { - return true; - } - else - { - in_transaction = false; - return false; - } -} - -bool CDatabase::BeginWriteTransaction() -{ - if (write_lock.get() == NULL) - { - transaction_read_lock.reset(new IScopedReadLock(single_user_mutex)); - } - - in_transaction = true; - if(Write("BEGIN IMMEDIATE;")) - { - return true; - } - else - { - in_transaction = false; - return false; - } -} - -bool CDatabase::EndTransaction(void) -{ - bool ret = Write("END;"); - in_transaction=false; - transaction_read_lock.reset(); - IScopedLock lock(lock_mutex); - bool waited=false; - while(*lock_count>0) - { - unlock_cond->wait(&lock); - waited=true; - } - if(waited) - { - Server->wait(50); - } - return ret; -} - -bool CDatabase::RollbackTransaction() -{ - bool ret = Write("ROLLBACK;"); - in_transaction = false; - transaction_read_lock.reset(); - IScopedLock lock(lock_mutex); - bool waited = false; - while (*lock_count>0) - { - unlock_cond->wait(&lock); - waited = true; - } - if (waited) - { - Server->wait(50); - } - return ret; -} - -IQuery* CDatabase::Prepare(std::string pQuery, bool autodestroy) -{ - IScopedReadLock lock(NULL); - - if (!in_transaction && write_lock.get()==NULL) - { - lock.relock(single_user_mutex); - } - - int prepare_tries = 0; -#ifdef SQLITE_PREPARE_RETRIES - prepare_tries = SQLITE_PREPARE_RETRIES; -#endif - - sqlite3_stmt *prepared_statement; - const char* tail; - int err; - bool transaction_lock=false; - while((err=sqlite3_prepare_v2(db, pQuery.c_str(), (int)pQuery.size(), &prepared_statement, &tail) )==SQLITE_LOCKED - || err==SQLITE_BUSY - || err==SQLITE_PROTOCOL - || (err!=SQLITE_OK && prepare_tries>0) ) - { - --prepare_tries; - - if(err==SQLITE_LOCKED) - { - if(!transaction_lock && LockForTransaction()) - { - transaction_lock=true; - if(!WaitForUnlock()) - Server->Log("DATABASE DEADLOCKED in CDatabase::Prepare", LL_ERROR); - } - } - else if(err== SQLITE_BUSY - || err==SQLITE_PROTOCOL) - { - if(!transaction_lock) - { - if(!isInTransaction() && LockForTransaction()) - { - transaction_lock=true; - } - sqlite3_busy_timeout(db, 10000); - } - else - { - Server->Log("DATABASE BUSY in CDatabase::Prepare", LL_ERROR); - } - } - else - { - Server->Log("Error preparing Query [" + pQuery + "]: " + sqlite3_errmsg(db)+". Retrying in 1s...", LL_ERROR); - Server->wait(1000); - } - } - - if(transaction_lock) - { - UnlockForTransaction(); - sqlite3_busy_timeout(db, 50); - } - - if( err!=SQLITE_OK ) - { - Server->Log("Error preparing Query ["+pQuery+"]: "+sqlite3_errmsg(db),LL_ERROR); - - if(err==SQLITE_IOERR) - { - Server->setFailBit(IServer::FAIL_DATABASE_IOERR); - } - if(err==SQLITE_CORRUPT) - { - Server->setFailBit(IServer::FAIL_DATABASE_CORRUPTED); - } - if (err ==SQLITE_FULL) - { - Server->setFailBit(IServer::FAIL_DATABASE_FULL); - } - - return NULL; - } - CQuery *q=new CQuery(pQuery, prepared_statement, this); - if( autodestroy ) - { - queries.push_back(q); - } - - return q; -} - -IQuery* CDatabase::Prepare(int id, std::string pQuery) -{ - IScopedReadLock lock(NULL); - - if (!in_transaction && write_lock.get()==NULL) - { - lock.relock(single_user_mutex); - } - - std::map::iterator iter=prepared_queries.find(id); - if( iter!=prepared_queries.end() ) - { - iter->second->Reset(); - return iter->second; - } - else - { - IQuery *q=Prepare(pQuery, false); - prepared_queries.insert(std::pair(id, q) ); - return q; - } -} - -void CDatabase::destroyQuery(IQuery *q) -{ - if(q==NULL) - { - return; - } - - for(size_t i=0;icreateMutex(); - un.cond=Server->createCondition(); - - rc = sqlite3_unlock_notify(db, unlock_notify_cb, (void *)&un); - - if( rc==SQLITE_OK ) - { - IScopedLock lock(un.mutex); - if( !un.fired ) - { - un.cond->wait(&lock); - } - } - - Server->destroy(un.mutex); - Server->destroy(un.cond); - - return rc==SQLITE_OK; -#else - return false; -#endif -} - -sqlite3 *CDatabase::getDatabase(void) -{ - return db; -} - -bool CDatabase::LockForTransaction(void) -{ - lock_mutex->Lock(); - ++*lock_count; - return true; -} - -void CDatabase::UnlockForTransaction(void) -{ - --*lock_count; - unlock_cond->notify_all(); - lock_mutex->Unlock(); -} - -bool CDatabase::isInTransaction(void) -{ - return in_transaction; -} - -bool CDatabase::Import(const std::string &pFile) -{ - IFile *file=Server->openFile(pFile, MODE_READ); - if(file==NULL) - return false; - - unsigned int r; - char buf[4096]; - std::string query; - int state=0; - do - { - r=file->Read(buf, 4096); - for(unsigned int i=0;i0); - - Server->destroy(file); - return true; -} - -bool CDatabase::Dump(const std::string &pFile) -{ - const char* db_fn = sqlite3_db_filename(db, NULL); - if (db_fn == NULL) - return false; - - ShellState cd = {}; - cd.openMode = 1; - cd.zDbFilename = db_fn; - cd.out=fopen(pFile.c_str(), "wb"); - if(cd.out==0) - { - return false; - } - - std::string cmd = ".dump"; - int rc = do_meta_command_r(&cmd[0], &cd); - - fclose(cd.out); - - if (cd.db != 0) - sqlite3_close(cd.db); - - return rc == SQLITE_OK; -} - -bool CDatabase::Recover(const std::string & pFile) -{ - const char* db_fn = sqlite3_db_filename(db, NULL); - if (db_fn == NULL) - return false; - - ShellState cd = {}; - cd.openMode = 1; - cd.zDbFilename = db_fn; - cd.out = fopen(pFile.c_str(), "wb"); - if (cd.out == 0) - { - return false; - } - - std::string cmd = ".recover"; - int rc = do_meta_command_r(&cmd[0], &cd); - - fclose(cd.out); - - if (cd.db != 0) - sqlite3_close(cd.db); - - return rc==SQLITE_OK; -} - -std::string CDatabase::getEngineName(void) -{ - #ifndef BDBPLUGIN - return "sqlite"; - #else - return "bdb"; - #endif -} - -void CDatabase::AttachDBs(void) -{ - for(size_t i=0;ibackupProgress(done*page_size, total*page_size); - - } while( rc==SQLITE_OK || rc==SQLITE_BUSY || rc==SQLITE_PROTOCOL || rc==SQLITE_LOCKED ); - - /* Release resources allocated by backup_init(). */ - (void)sqlite3_backup_finish(pBackup); - } - else - { - Server->Log("Opening backup connection failed", LL_ERROR); - } - rc = sqlite3_errcode(pBackupDB); - if(rc!=0) - { - Server->Log("Database backup failed with error code: "+convert(rc)+" err: "+sqlite3_errmsg(pBackupDB), LL_ERROR); - } - } - - /* Close the database connection opened on database file zFilename - ** and return the result of this function. */ - (void)sqlite3_close(pBackupDB); - return rc==0; -} - -bool CDatabase::Backup(const std::string &pFile, IBackupProgress* progress) -{ - std::string path=ExtractFilePath(pFile); - bool b=backup_db(pFile, "main", progress); - if(!b) - return false; - - for(size_t i=0;i. +**************************************************************************/ +#ifndef NO_SQLITE + +#if defined(_WIN32) || defined(WIN32) +#define _CRT_SECURE_NO_WARNINGS +#endif + +#include "vld.h" +#ifndef BDBPLUGIN +#include "Server.h" +#else +#ifdef LINUX +#include "bdbplugin/config.h" +#include DB_HEADER +#else +#include +#endif +#include "Interface/Server.h" +#endif +#include "Query.h" +#ifdef USE_SYSTEM_SQLITE +#include +#else +#include "sqlite/sqlite3.h" +#endif +#include "Interface/File.h" +#include +extern "C" +{ + #include "sqlite/shell.h" +} +#include "Database.h" +#include "stringtools.h" +#include + + +namespace +{ + size_t get_sqlite_cache_size() + { + std::string cache_size_str = Server->getServerParameter("sqlite_cache_size"); + + if(!cache_size_str.empty()) + { + return atoi(cache_size_str.c_str()); + } + else + { + return 2*1024; //2MB + } + } + + void errorLogCallback(void *pArg, int iErrCode, const char *zMsg) + { + switch (iErrCode) + { + case SQLITE_LOCKED: + case SQLITE_BUSY: + case SQLITE_SCHEMA: + return; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + case SQLITE_NOTICE_RECOVER_WAL: + Server->Log("SQLite: "+ std::string(zMsg) + " code: " + convert(iErrCode), LL_INFO); + break; + default: + Server->Log("SQLite: " + std::string(zMsg) + " errorcode: " + convert(iErrCode), LL_WARNING); + break; + } + } +} + + +struct UnlockNotification { + bool fired; + ICondition* cond; + IMutex *mutex; +}; + +static void unlock_notify_cb(void **apArg, int nArg) +{ + for(int i=0; imutex); + p->fired = true; + p->cond->notify_all(); + } +} + +CDatabase::~CDatabase() +{ +#ifndef NDEBUG + db_thread_id = Server->getThreadID(); +#endif + destroyAllQueries(); + for(std::map::iterator iter=prepared_queries.begin();iter!=prepared_queries.end();++iter) + { + CQuery *q=(CQuery*)iter->second; + delete q; + } + prepared_queries.clear(); + + sqlite3_close(db); +} + +bool CDatabase::Open(std::string pFile, const std::vector > &attach, + size_t allocation_chunk_size, ISharedMutex* p_single_user_mutex, IMutex* p_lock_mutex, + int* p_lock_count, ICondition *p_unlock_cond, const str_map& p_params) +{ + single_user_mutex = p_single_user_mutex; + lock_mutex = p_lock_mutex; + lock_count = p_lock_count; + unlock_cond = p_unlock_cond; + params = p_params; +#ifndef NDEBUG + db_thread_id = Server->getThreadID(); +#endif + + attached_dbs=attach; + in_transaction=false; + if( sqlite3_open(pFile.c_str(), &db) ) + { + Server->Log("Could not open db ["+pFile+"]"); + sqlite3_close(db); + db = NULL; + return false; + } + else + { + str_map::const_iterator it = params.find("synchronous"); + if (it != params.end()) + { + Write("PRAGMA synchronous="+it->second); + } + else + { + Write("PRAGMA synchronous=NORMAL"); + } + Write("PRAGMA foreign_keys = ON"); + Write("PRAGMA threads = 2"); + + it = params.find("wal_autocheckpoint"); + if (it != params.end()) + { + Write("PRAGMA wal_autocheckpoint=" + it->second); + + if (watoi(it->second)<=0) + { + int enable = 1; + sqlite3_file_control(db, NULL, SQLITE_FCNTL_PERSIST_WAL, &enable); +#ifdef SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE + int was_enabled = 0; + sqlite3_db_config(db, SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, 1, &was_enabled); +#endif + } + } + + it = params.find("page_size"); + if (it != params.end()) + { + Write("PRAGMA page_size=" + it->second); + } + else + { + Write("PRAGMA page_size=4096"); + } + + it = params.find("mmap_size"); + if (it != params.end()) + { + Write("PRAGMA mmap_size=" + it->second); + } + + if(allocation_chunk_size!=std::string::npos) + { + int chunk_size = static_cast(allocation_chunk_size); + sqlite3_file_control(db, NULL, SQLITE_FCNTL_CHUNK_SIZE, &chunk_size); + } + + static size_t sqlite_cache_size = get_sqlite_cache_size(); + Write("PRAGMA cache_size = -"+convert(sqlite_cache_size)); + + sqlite3_busy_timeout(db, c_sqlite_busy_timeout_default); + +#if defined(_DEBUG) || (!defined(_WIN32) && !defined(NDEBUG)) + if (Server->getRandomNumber() % 2 == 0) + { + Write("PRAGMA reverse_unordered_selects = ON"); + } +#endif + + AttachDBs(); + + return true; + } +} + +void CDatabase::initMutex(void) +{ + sqlite3_config(SQLITE_CONFIG_LOG, errorLogCallback, NULL); +} + +void CDatabase::destroyMutex(void) +{ +} + +db_results CDatabase::Read(std::string pQuery) +{ + assert_thread_id(); + //Server->Log("SQL Query(Read): "+pQuery, LL_DEBUG); + IQuery *q=Prepare(pQuery, false); + if(q!=NULL) + { + db_results ret=q->Read(); + delete ((CQuery*)q); + return ret; + } + return db_results(); +} + +bool CDatabase::Write(std::string pQuery) +{ + assert_thread_id(); + //Server->Log("SQL Query(Write): "+pQuery, LL_DEBUG); + IQuery *q=Prepare(pQuery, false); + if(q!=NULL) + { + bool b=q->Write(); + delete ((CQuery*)q); + return b; + } + else + { + return false; + } +} + +//ToDo: Cache Writings + +bool CDatabase::BeginReadTransaction() +{ + assert_thread_id(); + + if (write_lock.get() == NULL) + { + transaction_read_lock.reset(new IScopedReadLock(single_user_mutex)); + } + + in_transaction = true; + if(Write("BEGIN")) + { + return true; + } + else + { + in_transaction = false; + return false; + } +} + +bool CDatabase::BeginWriteTransaction() +{ + assert_thread_id(); + + if (write_lock.get() == NULL) + { + transaction_read_lock.reset(new IScopedReadLock(single_user_mutex)); + } + + in_transaction = true; + if(Write("BEGIN IMMEDIATE;")) + { + return true; + } + else + { + in_transaction = false; + return false; + } +} + +bool CDatabase::EndTransaction(void) +{ + assert_thread_id(); + + bool ret = Write("END;"); + in_transaction=false; + transaction_read_lock.reset(); + IScopedLock lock(lock_mutex); + bool waited=false; + while(*lock_count>0) + { + unlock_cond->wait(&lock); + waited=true; + } + if(waited) + { + Server->wait(50); + } + return ret; +} + +bool CDatabase::RollbackTransaction() +{ + assert_thread_id(); + + bool ret = Write("ROLLBACK;"); + in_transaction = false; + transaction_read_lock.reset(); + IScopedLock lock(lock_mutex); + bool waited = false; + while (*lock_count>0) + { + unlock_cond->wait(&lock); + waited = true; + } + if (waited) + { + Server->wait(50); + } + return ret; +} + +IQuery* CDatabase::Prepare(std::string pQuery, bool autodestroy) +{ + assert_thread_id(); + + IScopedReadLock lock(NULL); + + if (!in_transaction && write_lock.get()==NULL) + { + lock.relock(single_user_mutex); + } + + int prepare_tries = 0; +#ifdef SQLITE_PREPARE_RETRIES + prepare_tries = SQLITE_PREPARE_RETRIES; +#endif + + sqlite3_stmt *prepared_statement; + const char* tail; + int err; + bool transaction_lock=false; + while((err=sqlite3_prepare_v2(db, pQuery.c_str(), (int)pQuery.size(), &prepared_statement, &tail) )==SQLITE_LOCKED + || err==SQLITE_BUSY + || err==SQLITE_PROTOCOL + || (err!=SQLITE_OK && prepare_tries>0) ) + { + --prepare_tries; + + if(err==SQLITE_LOCKED) + { + if(!transaction_lock && LockForTransaction()) + { + transaction_lock=true; + if(!WaitForUnlock()) + Server->Log("DATABASE DEADLOCKED in CDatabase::Prepare", LL_ERROR); + } + } + else if(err== SQLITE_BUSY + || err==SQLITE_PROTOCOL) + { + if(!transaction_lock) + { + if(!isInTransaction() && LockForTransaction()) + { + transaction_lock=true; + } + sqlite3_busy_timeout(db, 10000); + } + else + { + Server->Log("DATABASE BUSY in CDatabase::Prepare", LL_ERROR); + } + } + else + { + Server->Log("Error preparing Query [" + pQuery + "]: " + sqlite3_errmsg(db)+". Retrying in 1s...", LL_ERROR); + Server->wait(1000); + } + } + + if(transaction_lock) + { + UnlockForTransaction(); + sqlite3_busy_timeout(db, 50); + } + + if( err!=SQLITE_OK ) + { + Server->Log("Error preparing Query ["+pQuery+"]: "+sqlite3_errmsg(db),LL_ERROR); + + if(err==SQLITE_IOERR) + { + Server->setFailBit(IServer::FAIL_DATABASE_IOERR); + } + if(err==SQLITE_CORRUPT) + { + Server->setFailBit(IServer::FAIL_DATABASE_CORRUPTED); + } + if (err ==SQLITE_FULL) + { + Server->setFailBit(IServer::FAIL_DATABASE_FULL); + } + + return NULL; + } + CQuery *q=new CQuery(pQuery, prepared_statement, this); + if( autodestroy ) + { + queries.push_back(q); + } + + return q; +} + +IQuery* CDatabase::Prepare(int id, std::string pQuery) +{ + assert_thread_id(); + + IScopedReadLock lock(NULL); + + if (!in_transaction && write_lock.get()==NULL) + { + lock.relock(single_user_mutex); + } + + std::map::iterator iter=prepared_queries.find(id); + if( iter!=prepared_queries.end() ) + { + iter->second->Reset(); + return iter->second; + } + else + { + IQuery *q=Prepare(pQuery, false); + prepared_queries.insert(std::pair(id, q) ); + return q; + } +} + +void CDatabase::destroyQuery(IQuery *q) +{ + assert_thread_id(); + + if(q==NULL) + { + return; + } + + for(size_t i=0;icreateMutex(); + un.cond=Server->createCondition(); + + rc = sqlite3_unlock_notify(db, unlock_notify_cb, (void *)&un); + + if( rc==SQLITE_OK ) + { + IScopedLock lock(un.mutex); + if( !un.fired ) + { + un.cond->wait(&lock); + } + } + + Server->destroy(un.mutex); + Server->destroy(un.cond); + + return rc==SQLITE_OK; +#else + return false; +#endif +} + +sqlite3 *CDatabase::getDatabase(void) +{ + return db; +} + +bool CDatabase::LockForTransaction(void) +{ + lock_mutex->Lock(); + ++*lock_count; + return true; +} + +void CDatabase::UnlockForTransaction(void) +{ + --*lock_count; + unlock_cond->notify_all(); + lock_mutex->Unlock(); +} + +bool CDatabase::isInTransaction(void) +{ + return in_transaction; +} + +bool CDatabase::Import(const std::string &pFile) +{ + IFile *file=Server->openFile(pFile, MODE_READ); + if(file==NULL) + return false; + + Write("PRAGMA foreign_keys = OFF"); + + unsigned int r; + char buf[4096]; + std::string query; + int state=0; + do + { + r=file->Read(buf, 4096); + for(unsigned int i=0;i0); + + Server->destroy(file); + return true; +} + +bool CDatabase::Dump(const std::string &pFile) +{ + assert_thread_id(); + + const char* db_fn = sqlite3_db_filename(db, NULL); + if (db_fn == NULL) + return false; + + FILE* out = fopen(pFile.c_str(), "wb"); + if (!out) + return false; + + ShellState* cd = shell_state_init(db_fn, out); + if (!cd) + return false; + + std::string cmd = ".dump"; + int rc = do_meta_command_r(&cmd[0], cd); + + shell_state_free(cd); + + return rc == SQLITE_OK; +} + +bool CDatabase::Recover(const std::string & pFile) +{ + const char* db_fn = sqlite3_db_filename(db, NULL); + if (db_fn == NULL) + return false; + + FILE* out = fopen(pFile.c_str(), "wb"); + if (!out) + return false; + + ShellState* cd = shell_state_init(db_fn, out); + if (!cd) + return false; + + std::string cmd = ".recover"; + int rc = do_meta_command_r(&cmd[0], cd); + + shell_state_free(cd); + + return rc==SQLITE_OK; +} + +std::string CDatabase::getEngineName(void) +{ + #ifndef BDBPLUGIN + return "sqlite"; + #else + return "bdb"; + #endif +} + +void CDatabase::AttachDBs(void) +{ + for(size_t i=0;ibackupProgress(done*page_size, total*page_size); + + } while( rc==SQLITE_OK || rc==SQLITE_BUSY || rc==SQLITE_PROTOCOL || rc==SQLITE_LOCKED ); + + /* Release resources allocated by backup_init(). */ + (void)sqlite3_backup_finish(pBackup); + } + else + { + Server->Log("Opening backup connection failed", LL_ERROR); + } + rc = sqlite3_errcode(pBackupDB); + if(rc!=0) + { + Server->Log("Database backup failed with error code: "+convert(rc)+" err: "+sqlite3_errmsg(pBackupDB), LL_ERROR); + } + } + + /* Close the database connection opened on database file zFilename + ** and return the result of this function. */ + (void)sqlite3_close(pBackupDB); + return rc==0; +} + +bool CDatabase::Backup(const std::string &pFile, IBackupProgress* progress) +{ + std::string path=ExtractFilePath(pFile); + bool b=backup_db(pFile, "main", progress); + if(!b) + return false; + + for(size_t i=0;igetThreadID() == db_thread_id); +} +#endif //NO_SQLITE diff --git a/Database.h b/Database.h index 65973e985..d3d7a856a 100644 --- a/Database.h +++ b/Database.h @@ -71,8 +71,13 @@ class CDatabase : public IDatabaseInt ISharedMutex* getSingleUseMutex(); private: + void assert_thread_id(); + DATABASE_ID database_id; +#ifndef NDEBUG + THREAD_ID db_thread_id; +#endif bool backup_db(const std::string &pFile, const std::string &pDB, IBackupProgress* progress); diff --git a/Interface/Pipe.h b/Interface/Pipe.h index c44729d8c..dc4cdb491 100644 --- a/Interface/Pipe.h +++ b/Interface/Pipe.h @@ -45,6 +45,43 @@ class IPipe : public IObject virtual void resetTransferedBytes(void)=0; virtual _i64 getRealTransferredBytes() { return 0; } + + virtual void setUsageString(const std::string& str) = 0; + + + enum CompressionMem + { + Compression_LowMem, + Compression_HighMem + }; + + struct SCompressionSettings + { + SCompressionSettings() + : mem(Compression_LowMem), + n_threads(0), + buffer_size(0), + send_timeout(10000), + adaptive_compression(false), + adaptive_comp_flush_timeout(60000) + {} + + CompressionMem mem; + size_t n_threads; + size_t buffer_size; + int64 send_timeout; + bool adaptive_compression; + int64 adaptive_comp_flush_timeout; + }; + + virtual bool setCompressionSettings(const SCompressionSettings& params) = 0; + + enum SocketOption + { + SocketOption_NoDelay = 1 + }; + + virtual bool setOption(const SocketOption opt) = 0; }; #endif //IPIPE_H diff --git a/Interface/Server.h b/Interface/Server.h index 838959a4f..4e9df81fb 100644 --- a/Interface/Server.h +++ b/Interface/Server.h @@ -152,6 +152,7 @@ class IServer virtual void DisconnectStream(IPipe *pipe)=0; virtual std::string LookupHostname(const std::string& pIp)=0; + virtual std::vector getStreamPipeList() = 0; virtual bool RegisterPluginPerThreadModel(IPluginMgr *pPluginMgr, std::string pName)=0; virtual bool RegisterPluginThreadsafeModel(IPluginMgr *pPluginMgr, std::string pName)=0; diff --git a/Interface/SettingsReader.h b/Interface/SettingsReader.h index 63b09ed7e..ef157fa4f 100644 --- a/Interface/SettingsReader.h +++ b/Interface/SettingsReader.h @@ -10,11 +10,13 @@ class ISettingsReader : public IObject public: virtual bool getValue(std::string key, std::string *value)=0; - virtual std::string getValue(std::string key, std::string def)=0; + virtual std::string getValue(const std::string& key, const char* def) = 0; + virtual std::string getValue(std::string key, const std::string& def)=0; virtual std::string getValue(std::string key)=0; virtual int getValue(std::string key, int def)=0; virtual float getValue(std::string key, float def)=0; virtual int64 getValue(std::string key, int64 def)=0; + virtual bool getValue(const std::string& key, const bool def) = 0; virtual std::vector getKeys() = 0; }; diff --git a/Makefile.am_client b/Makefile.am_client index e131f708f..1d878e627 100644 --- a/Makefile.am_client +++ b/Makefile.am_client @@ -1,9 +1,11 @@ ACLOCAL_AMFLAGS = -I m4 sbin_PROGRAMS = urbackupclientbackend -if WITH_GUI_CLIENT -bin_PROGRAMS = urbackupclientctl urbackupclientgui blockalign -else bin_PROGRAMS = urbackupclientctl blockalign +if WITH_GUI_CLIENT +bin_PROGRAMS += urbackupclientgui +endif +if WITH_DMTOOL +sbin_PROGRAMS += urbackupclient_dmsnaptool endif urbackupclientbackend_SOURCES = AcceptThread.cpp Client.cpp Database.cpp Query.cpp SelectThread.cpp Server.cpp ServerLinux.cpp ServiceAcceptor.cpp ServiceWorker.cpp SessionMgr.cpp StreamPipe.cpp Template.cpp WorkerThread.cpp main.cpp md5.cpp stringtools.cpp libfastcgi/fastcgi.cpp Mutex_lin.cpp LoadbalancerClient.cpp DBSettingsReader.cpp file_common.cpp file_fstream.cpp file_linux.cpp FileSettingsReader.cpp LookupService.cpp SettingsReader.cpp Table.cpp OutputStream.cpp ThreadPool.cpp MemoryPipe.cpp Condition_lin.cpp MemorySettingsReader.cpp sqlite/shell.c SQLiteFactory.cpp PipeThrottler.cpp mt19937ar.cpp DatabaseCursor.cpp SharedMutex_lin.cpp StaticPluginRegistration.cpp common/data.cpp common/adler32.cpp OpenSSLPipe.cpp @@ -19,9 +21,9 @@ endif urbackupclientbackend_SOURCES += cryptoplugin/dllmain.cpp cryptoplugin/AESDecryption.cpp cryptoplugin/CryptoFactory.cpp cryptoplugin/pluginmgr.cpp cryptoplugin/AESEncryption.cpp cryptoplugin/ZlibCompression.cpp cryptoplugin/ZlibDecompression.cpp cryptoplugin/AESGCMDecryption.cpp cryptoplugin/AESGCMEncryption.cpp cryptoplugin/ECDHKeyExchange.cpp -urbackupclientbackend_SOURCES += fsimageplugin/dllmain.cpp fsimageplugin/filesystem.cpp fsimageplugin/FSImageFactory.cpp fsimageplugin/pluginmgr.cpp fsimageplugin/vhdfile.cpp fsimageplugin/fs/ntfs.cpp fsimageplugin/fs/unknown.cpp fsimageplugin/CompressedFile.cpp fsimageplugin/LRUMemCache.cpp fsimageplugin/cowfile.cpp fsimageplugin/FileWrapper.cpp fsimageplugin/ClientBitmap.cpp fsimageplugin/partclone.cpp +urbackupclientbackend_SOURCES += fsimageplugin/dllmain.cpp fsimageplugin/filesystem.cpp fsimageplugin/FSImageFactory.cpp fsimageplugin/pluginmgr.cpp fsimageplugin/vhdfile.cpp fsimageplugin/vhdxfile.cpp fsimageplugin/fs/ntfs.cpp fsimageplugin/fs/unknown.cpp fsimageplugin/CompressedFile.cpp fsimageplugin/LRUMemCache.cpp fsimageplugin/cowfile.cpp fsimageplugin/FileWrapper.cpp fsimageplugin/ClientBitmap.cpp fsimageplugin/partclone.cpp -urbackupclientbackend_SOURCES += urbackupclient/dllmain.cpp urbackupclient/clientdao.cpp urbackupclient/client.cpp urbackupclient/ClientService.cpp urbackupclient/ClientSend.cpp urbackupclient/client_restore.cpp urbackupclient/ServerIdentityMgr.cpp urbackupclient/ClientServiceCMD.cpp urbackupclient/ImageThread.cpp urbackupclient/InternetClient.cpp urbackupclient/file_permissions.cpp urbackupclient/lin_ver.cpp urbackupclient/lin_tokens.cpp urbackupclient/common_tokens.cpp urbackupclient/FileMetadataDownloadThread.cpp urbackupclient/RestoreFiles.cpp urbackupclient/RestoreDownloadThread.cpp urbackupclient/TokenCallback.cpp common/miniz.c urbackupclient/cmdline_preprocessor.cpp urbackupclient/ParallelHash.cpp urbackupclient/ClientHash.cpp +urbackupclientbackend_SOURCES += urbackupclient/dllmain.cpp urbackupclient/clientdao.cpp urbackupclient/client.cpp urbackupclient/ClientService.cpp urbackupclient/ClientSend.cpp urbackupclient/client_restore.cpp urbackupclient/ServerIdentityMgr.cpp urbackupclient/ClientServiceCMD.cpp urbackupclient/ImageThread.cpp urbackupclient/InternetClient.cpp urbackupclient/file_permissions.cpp urbackupclient/lin_ver.cpp urbackupclient/lin_tokens.cpp urbackupclient/common_tokens.cpp urbackupclient/FileMetadataDownloadThread.cpp urbackupclient/RestoreFiles.cpp urbackupclient/RestoreDownloadThread.cpp urbackupclient/TokenCallback.cpp common/miniz.c urbackupclient/cmdline_preprocessor.cpp urbackupclient/ParallelHash.cpp urbackupclient/ClientHash.cpp urbackupclient/SambaService.cpp urbackupclientbackend_SOURCES += fileservplugin/dllmain.cpp fileservplugin/bufmgr.cpp fileservplugin/CClientThread.cpp fileservplugin/CriticalSection.cpp fileservplugin/CTCPFileServ.cpp fileservplugin/CUDPThread.cpp fileservplugin/FileServ.cpp fileservplugin/FileServFactory.cpp fileservplugin/log.cpp fileservplugin/main.cpp fileservplugin/map_buffer.cpp fileservplugin/pluginmgr.cpp fileservplugin/ChunkSendThread.cpp fileservplugin/PipeFile.cpp fileservplugin/PipeSessions.cpp fileservplugin/PipeFileUnix.cpp fileservplugin/PipeFileBase.cpp fileservplugin/FileMetadataPipe.cpp fileservplugin/PipeFileTar.cpp fileservplugin/PipeFileExt.cpp @@ -121,6 +123,15 @@ if !WITH_ASSERTIONS urbackupclientctl_CPPFLAGS=-DNDEBUG endif +if WITH_DMTOOL +urbackupclient_dmsnaptool_SOURCES = dmsnaptool/main.cpp stringtools.cpp +urbackupclient_dmsnaptool_LDADD = -ldevmapper +urbackupclient_dmsnaptool_LDFLAGS = +if !WITH_ASSERTIONS +urbackupclient_dmsnaptool_CPPFLAGS=-DNDEBUG +endif +endif + if MACOSX urbackupclientgui_osx_SOURCES = client/osxutils.mm else @@ -270,7 +281,7 @@ cryptoplugin_headers = cryptoplugin/AESEncryption.h cryptoplugin/AESDecryption.h fileservplugin_headers = fileservplugin/bufmgr.h fileservplugin/CUDPThread.h fileservplugin/FileServFactory.h fileservplugin/IFileServ.h fileservplugin/packet_ids.h fileservplugin/socket_header.h fileservplugin/CriticalSection.h fileservplugin/FileServ.h fileservplugin/log.h fileservplugin/pluginmgr.h fileservplugin/CClientThread.h fileservplugin/CTCPFileServ.h fileservplugin/IFileServFactory.h fileservplugin/map_buffer.h fileservplugin/settings.h fileservplugin/types.h fileservplugin/chunk_settings.h fileservplugin/ChunkSendThread.h fileservplugin/PipeFile.h fileservplugin/PipeSessions.h fileservplugin/PipeFileBase.h fileservplugin/IPermissionCallback.h fileservplugin/FileMetadataPipe.h fileservplugin/PipeFileTar.h fileservplugin/PipeFileExt.h fileservplugin/IPipeFileExt.h -fsimageplugin_headers = fsimageplugin/filesystem.h fsimageplugin/FSImageFactory.h fsimageplugin/IFilesystem.h fsimageplugin/IFSImageFactory.h fsimageplugin/IVHDFile.h fsimageplugin/pluginmgr.h fsimageplugin/vhdfile.h fsimageplugin/fs/ntfs.h fsimageplugin/fs/unknown.h fsimageplugin/CompressedFile.h fsimageplugin/LRUMemCache.h fsimageplugin/cowfile.h fsimageplugin/FileWrapper.h fsimageplugin/ClientBitmap.h common/miniz.h fsimageplugin/partclone.h +fsimageplugin_headers = fsimageplugin/filesystem.h fsimageplugin/FSImageFactory.h fsimageplugin/IFilesystem.h fsimageplugin/IFSImageFactory.h fsimageplugin/IVHDFile.h fsimageplugin/pluginmgr.h fsimageplugin/vhdfile.h fsimageplugin/vhdxfile.h fsimageplugin/fs/ntfs.h fsimageplugin/fs/unknown.h fsimageplugin/CompressedFile.h fsimageplugin/LRUMemCache.h fsimageplugin/cowfile.h fsimageplugin/FileWrapper.h fsimageplugin/ClientBitmap.h common/miniz.h fsimageplugin/partclone.h urbackupclientctl_headers = clientctl/Connector.h clientctl/tcpstack.h clientctl/json/json.h clientctl/json/json-forwards.h @@ -280,7 +291,7 @@ else client_headers = endif -urbackupclient_headers = urbackupclient/DirectoryWatcherThread.h urbackupcommon/os_functions.h urbackupclient/ChangeJournalWatcher.h urbackupcommon/sha2/sha2.h urbackupclient/database.h urbackupcommon/escape.h urbackupclient/ClientSend.h urbackupclient/clientdao.h urbackupclient/client.h urbackupclient/ClientService.h fileservplugin/IFileServFactory.h fileservplugin/IFileServ.h common/data.h urbackupcommon/fileclient/tcpstack.h urbackupcommon/capa_bits.h urbackupclient/ServerIdentityMgr.h urbackupcommon/bufmgr.h urbackupcommon/CompressedPipe.h urbackupclient/ImageThread.h urbackupclient/InternetClient.h urbackupcommon/InternetServicePipe2.h urbackupcommon/settingslist.h cryptoplugin/IZlibCompression.h cryptoplugin/IZlibDecompression.h cryptoplugin/ICryptoFactory.h cryptoplugin/IAESDecryption.h cryptoplugin/IAESEncryption.h urbackupcommon/internet_pipe_capabilities.h urbackupcommon/settings.h urbackupcommon/fileclient/socket_header.h urbackupcommon/mbrdata.h urbackupcommon/InternetServiceIDs.h urbackupcommon/json.h urbackupclient/file_permissions.h urbackupclient/lin_ver.h urbackupcommon/glob.h urbackupclient/tokens.h urbackupclient/FileMetadataDownloadThread.h urbackupclient/RestoreFiles.h urbackupcommon/chunk_hasher.h common/adler32.h urbackupcommon/fileclient/FileClient.h urbackupcommon/fileclient/FileClientChunked.h urbackupcommon/file_metadata.h urbackupcommon/filelist_utils.h urbackupclient/RestoreDownloadThread.h urbackupclient/TokenCallback.h urbackupcommon/CompressedPipe2.h urbackupcommon/server_compat.h urbackupcommon/fileclient/packet_ids.h urbackupcommon/InternetServicePipe.h urbackupclient/backup_client_db.h urbackupcommon/SparseFile.h urbackupcommon/ExtentIterator.h urbackupcommon/TreeHash.h urbackupcommon/WalCheckpointThread.h common/miniz.h urbackupclient/ParallelHash.h urbackupclient/ClientHash.h urbackupcommon/CompressedPipeZstd.h urbackupclient/lin_sysvol.h urbackupcommon/WebSocketPipe.h +urbackupclient_headers = urbackupclient/DirectoryWatcherThread.h urbackupcommon/os_functions.h urbackupclient/ChangeJournalWatcher.h urbackupcommon/sha2/sha2.h urbackupclient/database.h urbackupcommon/escape.h urbackupclient/ClientSend.h urbackupclient/clientdao.h urbackupclient/client.h urbackupclient/ClientService.h fileservplugin/IFileServFactory.h fileservplugin/IFileServ.h common/data.h urbackupcommon/fileclient/tcpstack.h urbackupcommon/capa_bits.h urbackupclient/ServerIdentityMgr.h urbackupcommon/bufmgr.h urbackupcommon/CompressedPipe.h urbackupclient/ImageThread.h urbackupclient/InternetClient.h urbackupcommon/InternetServicePipe2.h urbackupcommon/settingslist.h cryptoplugin/IZlibCompression.h cryptoplugin/IZlibDecompression.h cryptoplugin/ICryptoFactory.h cryptoplugin/IAESDecryption.h cryptoplugin/IAESEncryption.h urbackupcommon/internet_pipe_capabilities.h urbackupcommon/settings.h urbackupcommon/fileclient/socket_header.h urbackupcommon/mbrdata.h urbackupcommon/InternetServiceIDs.h urbackupcommon/json.h urbackupclient/file_permissions.h urbackupclient/lin_ver.h urbackupcommon/glob.h urbackupclient/tokens.h urbackupclient/FileMetadataDownloadThread.h urbackupclient/RestoreFiles.h urbackupcommon/chunk_hasher.h common/adler32.h urbackupcommon/fileclient/FileClient.h urbackupcommon/fileclient/FileClientChunked.h urbackupcommon/file_metadata.h urbackupcommon/filelist_utils.h urbackupclient/RestoreDownloadThread.h urbackupclient/TokenCallback.h urbackupcommon/CompressedPipe2.h urbackupcommon/server_compat.h urbackupcommon/fileclient/packet_ids.h urbackupcommon/InternetServicePipe.h urbackupclient/backup_client_db.h urbackupcommon/SparseFile.h urbackupcommon/ExtentIterator.h urbackupcommon/TreeHash.h urbackupcommon/WalCheckpointThread.h common/miniz.h urbackupclient/ParallelHash.h urbackupclient/ClientHash.h urbackupcommon/CompressedPipeZstd.h urbackupclient/lin_sysvol.h urbackupcommon/WebSocketPipe.h urbackupclient/SambaService.h tclap_headers = \ @@ -346,6 +357,6 @@ zstd_headers = \ noinst_HEADERS=SessionMgr.h WorkerThread.h Helper_win32.h Database.h defaults.h ServiceAcceptor.h Query.h SettingsReader.h file.h file_memory.h MemorySettingsReader.h Condition_lin.h LookupService.h Template.h types.h DBSettingsReader.h stringtools.h ThreadPool.h libs.h vld_.h ServiceWorker.h StreamPipe.h LoadbalancerClient.h socket_header.h FileSettingsReader.h SelectThread.h md5.h vld.h Table.h Client.h MemoryPipe.h Mutex_lin.h AcceptThread.h OutputStream.h Server.h Interface/SessionMgr.h Interface/Service.h Interface/PluginMgr.h Interface/Database.h Interface/Pipe.h Interface/CustomClient.h Interface/User.h Interface/Query.h Interface/SettingsReader.h Interface/Types.h Interface/Template.h Interface/ThreadPool.h Interface/Mutex.h Interface/File.h Interface/Condition.h Interface/Table.h Interface/Plugin.h Interface/Thread.h Interface/Action.h Interface/Object.h Interface/OutputStream.h Interface/Server.h libfastcgi/fastcgi.hpp sqlite/sqlite3.h sqlite/sqlite3ext.h utf8/utf8.h utf8/utf8/checked.h utf8/utf8/core.h utf8/utf8/unchecked.h cryptoplugin/ICryptoFactory.h cryptoplugin/IAESEncryption.h cryptoplugin/IAESDecryption.h Interface/DatabaseFactory.h Interface/DatabaseInt.h sqlite/shell.h SQLiteFactory.h PipeThrottler.h Interface/PipeThrottler.h mt19937ar.h DatabaseCursor.h Interface/DatabaseCursor.h Interface/WebSocket.h client_version.h Interface/SharedMutex.h SharedMutex_lin.h StaticPluginRegistration.h common/bitmap.h OpenSSLPipe.h $(cryptoplugin_headers) $(fileservplugin_headers) $(fsimageplugin_headers) $(urbackupclientctl_headers) $(client_headers) $(tclap_headers) $(urbackupclient_headers) $(cryptopp_headers) $(blockalign_headers) $(zstd_headers) -EXTRA_DIST_GUI = client/info.txt client/data/backup-bad.xpm client/data/backup-ok.xpm client/data/backup-progress.xpm client/data/backup-progress-pause.xpm client/data/backup-no-server.xpm client/data/backup-no-recent.xpm client/data/backup-indexing.xpm client/data/logo1.png client/data/lang/it/urbackup.mo client/data/lang/pl/urbackup.mo client/data/lang/pt_BR/urbackup.mo client/data/lang/sk/urbackup.mo client/data/lang/zh_TW/urbackup.mo client/data/lang/zh_CN/urbackup.mo client/data/lang/de/urbackup.mo client/data/lang/es/urbackup.mo client/data/lang/fr/urbackup.mo client/data/lang/ru/urbackup.mo client/data/lang/uk/urbackup.mo client/data/lang/da/urbackup.mo client/data/lang/nl/urbackup.mo client/data/lang/fa/urbackup.mo client/data/lang/cs/urbackup.mo client/gui/GUISetupWizard.h client/SetupWizard.h +EXTRA_DIST_GUI = client/info.txt client/data/backup-bad.xpm client/data/backup-ok.xpm client/data/backup-progress.xpm client/data/backup-progress-pause.xpm client/data/backup-no-server.xpm client/data/backup-no-recent.xpm client/data/backup-indexing.xpm client/data/logo1.png client/data/lang/it/urbackup.mo client/data/lang/pl/urbackup.mo client/data/lang/pt_BR/urbackup.mo client/data/lang/sk/urbackup.mo client/data/lang/zh_TW/urbackup.mo client/data/lang/zh_CN/urbackup.mo client/data/lang/de/urbackup.mo client/data/lang/es/urbackup.mo client/data/lang/fr/urbackup.mo client/data/lang/ru/urbackup.mo client/data/lang/uk/urbackup.mo client/data/lang/da/urbackup.mo client/data/lang/nl/urbackup.mo client/data/lang/fa/urbackup.mo client/data/lang/cs/urbackup.mo client/gui/GUISetupWizard.h client/SetupWizard.h client/fa-copy.png client/fa-home.png client/fa-lock.png client/fa-road.png -EXTRA_DIST=docs/urbackupclientbackend.1 init.d_client init.d_client_rh defaults_client $(EXTRA_DIST_GUI) tclap/COPYING tclap/AUTHORS urbackupclientbackend-debian.service urbackupclientbackend-redhat.service urbackupclient/backup_scripts/list urbackupclient/backup_scripts/mariadbdump.conf urbackupclient/backup_scripts/mariadbdump urbackupclient/backup_scripts/postgresqldump.conf urbackupclient/backup_scripts/postgresqldump urbackupclient/backup_scripts/postgresbase urbackupclient/backup_scripts/postgresqlprebackup urbackupclient/backup_scripts/postgresqlpostbackup urbackupclient/backup_scripts/setup-postgresbackup urbackupclient/backup_scripts/postgresbase.conf urbackupclient/backup_scripts/mariadbxtrabackup.conf urbackupclient/backup_scripts/mariadbxtrabackup urbackupclient/backup_scripts/mariadbprebackup urbackupclient/backup_scripts/mariadbpostbackup urbackupclient/backup_scripts/setup-mariadbbackup urbackupclient/backup_scripts/restore-mariadbbackup client/version.txt client/data/urbackup_ecdsa409k1.pub linux_snapshot/btrfs_create_filesystem_snapshot linux_snapshot/btrfs_remove_filesystem_snapshot linux_snapshot/dattobd_create_snapshot linux_snapshot/dattobd_remove_snapshot linux_snapshot/lvm_create_filesystem_snapshot linux_snapshot/lvm_remove_filesystem_snapshot client/data/updates_h.dat +EXTRA_DIST=docs/urbackupclientbackend.1 init.d_client init.d_client_rh defaults_client $(EXTRA_DIST_GUI) tclap/COPYING tclap/AUTHORS urbackupclientbackend-debian.service urbackupclientbackend-redhat.service urbackupclient/backup_scripts/list urbackupclient/backup_scripts/list_incr urbackupclient/backup_scripts/mariadbdump.conf urbackupclient/backup_scripts/mariadbdump urbackupclient/backup_scripts/postgresqldump.conf urbackupclient/backup_scripts/postgresqldump urbackupclient/backup_scripts/postgresbase urbackupclient/backup_scripts/postgresqlprebackup urbackupclient/backup_scripts/postgresqlpostbackup urbackupclient/backup_scripts/setup-postgresbackup urbackupclient/backup_scripts/postgresbase.conf urbackupclient/backup_scripts/mariadbxtrabackup.conf urbackupclient/backup_scripts/mariadbxtrabackup urbackupclient/backup_scripts/mariadbxtrabackup_incr urbackupclient/backup_scripts/mariadbprebackup urbackupclient/backup_scripts/mariadbpostbackup urbackupclient/backup_scripts/setup-mariadbbackup urbackupclient/backup_scripts/restore-mariadbbackup client/version.txt client/data/urbackup_ecdsa409k1.pub linux_snapshot/btrfs_create_filesystem_snapshot linux_snapshot/btrfs_remove_filesystem_snapshot linux_snapshot/dattobd_create_snapshot linux_snapshot/dattobd_remove_snapshot linux_snapshot/lvm_create_filesystem_snapshot linux_snapshot/lvm_remove_filesystem_snapshot client/data/updates_h.dat linux_snapshot/dm_create_snapshot linux_snapshot/dm_remove_snapshot linux_snapshot/dm_create_volume_snapshot linux_snapshot/dm_remove_volume_snapshot linux_snapshot/dm_create_snapshot_common linux_snapshot/dm_remove_snapshot_common diff --git a/Makefile.am_server b/Makefile.am_server index cb1501a06..279163e21 100644 --- a/Makefile.am_server +++ b/Makefile.am_server @@ -6,7 +6,7 @@ if WITH_EMBEDDED_SQLITE3 urbackupsrv_SOURCES += sqlite/sqlite3.c endif -urbackupsrv_SOURCES += fsimageplugin/dllmain.cpp fsimageplugin/filesystem.cpp fsimageplugin/FSImageFactory.cpp fsimageplugin/pluginmgr.cpp fsimageplugin/vhdfile.cpp fsimageplugin/fs/ntfs.cpp fsimageplugin/fs/unknown.cpp fsimageplugin/CompressedFile.cpp fsimageplugin/LRUMemCache.cpp fsimageplugin/cowfile.cpp fsimageplugin/FileWrapper.cpp fsimageplugin/ClientBitmap.cpp fsimageplugin/partclone.cpp +urbackupsrv_SOURCES += fsimageplugin/dllmain.cpp fsimageplugin/filesystem.cpp fsimageplugin/FSImageFactory.cpp fsimageplugin/pluginmgr.cpp fsimageplugin/vhdfile.cpp fsimageplugin/vhdxfile.cpp fsimageplugin/fs/ntfs.cpp fsimageplugin/fs/unknown.cpp fsimageplugin/CompressedFile.cpp fsimageplugin/LRUMemCache.cpp fsimageplugin/cowfile.cpp fsimageplugin/FileWrapper.cpp fsimageplugin/ClientBitmap.cpp fsimageplugin/partclone.cpp urbackupsrv_SOURCES += urbackupcommon/os_functions_lin.cpp urbackupcommon/sha2/sha2.cpp urbackupcommon/fileclient/FileClient.cpp urbackupcommon/fileclient/tcpstack.cpp urbackupcommon/escape.cpp urbackupcommon/bufmgr.cpp urbackupcommon/json.cpp urbackupcommon/CompressedPipe.cpp urbackupcommon/InternetServicePipe2.cpp urbackupcommon/settingslist.cpp urbackupcommon/fileclient/FileClientChunked.cpp urbackupcommon/InternetServicePipe.cpp urbackupcommon/filelist_utils.cpp urbackupcommon/file_metadata.cpp urbackupcommon/glob.cpp urbackupcommon/chunk_hasher.cpp urbackupcommon/CompressedPipe2.cpp urbackupcommon/SparseFile.cpp urbackupcommon/ExtentIterator.cpp urbackupcommon/TreeHash.cpp @@ -228,7 +228,7 @@ cryptoplugin_headers = cryptoplugin/AESEncryption.h cryptoplugin/AESDecryption.h fileservplugin_headers = fileservplugin/bufmgr.h fileservplugin/CUDPThread.h fileservplugin/FileServFactory.h fileservplugin/IFileServ.h fileservplugin/packet_ids.h fileservplugin/socket_header.h fileservplugin/CriticalSection.h fileservplugin/FileServ.h fileservplugin/log.h fileservplugin/pluginmgr.h fileservplugin/CClientThread.h fileservplugin/CTCPFileServ.h fileservplugin/IFileServFactory.h fileservplugin/map_buffer.h fileservplugin/settings.h fileservplugin/types.h fileservplugin/chunk_settings.h fileservplugin/ChunkSendThread.h fileservplugin/PipeFile.h fileservplugin/PipeSessions.h fileservplugin/PipeFileBase.h fileservplugin/IPermissionCallback.h fileservplugin/FileMetadataPipe.h fileservplugin/PipeFileTar.h fileservplugin/PipeFileExt.h -fsimageplugin_headers = fsimageplugin/filesystem.h fsimageplugin/FSImageFactory.h fsimageplugin/IFilesystem.h fsimageplugin/IFSImageFactory.h fsimageplugin/IVHDFile.h fsimageplugin/pluginmgr.h fsimageplugin/vhdfile.h fsimageplugin/fs/ntfs.h fsimageplugin/fs/unknown.h fsimageplugin/CompressedFile.h fsimageplugin/LRUMemCache.h common/miniz.h fsimageplugin/cowfile.h fsimageplugin/FileWrapper.h fsimageplugin/ClientBitmap.h fsimageplugin/partclone.h +fsimageplugin_headers = fsimageplugin/filesystem.h fsimageplugin/FSImageFactory.h fsimageplugin/IFilesystem.h fsimageplugin/IFSImageFactory.h fsimageplugin/IVHDFile.h fsimageplugin/pluginmgr.h fsimageplugin/vhdfile.h fsimageplugin/vhdxfile.h fsimageplugin/fs/ntfs.h fsimageplugin/fs/unknown.h fsimageplugin/CompressedFile.h fsimageplugin/LRUMemCache.h common/miniz.h fsimageplugin/cowfile.h fsimageplugin/FileWrapper.h fsimageplugin/ClientBitmap.h fsimageplugin/partclone.h tclap_headers = \ tclap/CmdLineInterface.h \ @@ -290,6 +290,6 @@ zstd_headers = \ luaplugin_headers = luaplugin/ILuaInterpreter.h luaplugin/LuaInterpreter.h luaplugin/pluginmgr.h luaplugin/src/* luaplugin/lua/dkjson_lua.h -noinst_HEADERS=SessionMgr.h WorkerThread.h Helper_win32.h Database.h defaults.h ServiceAcceptor.h Query.h SettingsReader.h file.h file_memory.h MemorySettingsReader.h Condition_lin.h LookupService.h Template.h types.h DBSettingsReader.h stringtools.h ThreadPool.h libs.h vld_.h ServiceWorker.h StreamPipe.h LoadbalancerClient.h socket_header.h FileSettingsReader.h SelectThread.h md5.h vld.h Table.h Client.h MemoryPipe.h Mutex_lin.h AcceptThread.h OutputStream.h Server.h Interface/SessionMgr.h Interface/Service.h Interface/PluginMgr.h Interface/Database.h Interface/Pipe.h Interface/CustomClient.h Interface/User.h Interface/Query.h Interface/SettingsReader.h Interface/Types.h Interface/Template.h Interface/ThreadPool.h Interface/Mutex.h Interface/File.h Interface/Condition.h Interface/Table.h Interface/Plugin.h Interface/Thread.h Interface/Action.h Interface/Object.h Interface/OutputStream.h Interface/Server.h libfastcgi/fastcgi.hpp sqlite/sqlite3.h sqlite/sqlite3ext.h utf8/utf8.h utf8/utf8/checked.h utf8/utf8/core.h utf8/utf8/unchecked.h cryptoplugin/ICryptoFactory.h cryptoplugin/IAESEncryption.h cryptoplugin/IAESDecryption.h Interface/DatabaseFactory.h Interface/DatabaseInt.h SQLiteFactory.h sqlite/shell.h PipeThrottler.h Interface/PipeThrottler.h mt19937ar.h DatabaseCursor.h Interface/DatabaseCursor.h Interface/SharedMutex.h Interface/WebSocket.h SharedMutex_lin.h httpserver/HTTPAction.h httpserver/HTTPClient.h httpserver/HTTPFile.h httpserver/HTTPProxy.h httpserver/HTTPService.h httpserver/IndexFiles.h httpserver/MIMEType.h httpserver/HTTPSocket.h urbackupserver/server_ping.h urbackupserver/server_cleanup.h urbackupcommon/os_functions.h urbackupcommon/json.h urbackupserver/serverinterface/helper.h urbackupserver/serverinterface/action_header.h urbackupserver/serverinterface/actions.h urbackupserver/server_writer.h urbackupcommon/settings.h urbackupserver/server_settings.h urbackupserver/zero_hash.h urbackupserver/server_update.h urbackupserver/server_log.h urbackupserver/server_hash.h urbackupserver/server_status.h urbackupcommon/bufmgr.h urbackupserver/server_update_stats.h urbackupcommon/sha2/sha2.h urbackupcommon/fileclient/FileClient.h common/data.h urbackupcommon/fileclient/socket_header.h urbackupcommon/fileclient/tcpstack.h urbackupcommon/fileclient/packet_ids.h urbackupserver/database.h urbackupserver/mbr_code.h urbackupserver/action_header.h urbackupcommon/escape.h urbackupserver/server.h urbackupserver/server_running.h urbackupserver/server_prepare_hash.h urbackupserver/actions.h urbackupserver/server_channel.h urbackupserver/ClientMain.h urbackupserver/treediff/TreeDiff.h urbackupserver/treediff/TreeNode.h urbackupserver/treediff/TreeReader.h fileservplugin/IFileServFactory.h fileservplugin/IFileServ.h urlplugin/IUrlFactory.h urbackupcommon/capa_bits.h cryptoplugin/ICryptoFactory.h urbackupcommon/fileclient/FileClientChunked.h urbackupserver/ChunkPatcher.h urbackupcommon/CompressedPipe.h urbackupcommon/InternetServicePipe.h urbackupcommon/InternetServicePipe2.h urbackupcommon/InternetServiceIDs.h urbackupserver/InternetServiceConnector.h md5.h urbackupcommon/settingslist.h urbackupserver/server_archive.h cryptoplugin/IZlibCompression.h cryptoplugin/IZlibDecompression.h cryptoplugin/ICryptoFactory.h cryptoplugin/IAESEncryption.h cryptoplugin/IAESDecryption.h fileservplugin/chunk_settings.h urbackupcommon/internet_pipe_capabilities.h urbackupcommon/mbrdata.h urbackupserver/filedownload.h urbackupserver/snapshot_helper.h urbackupserver/apps/cleanup_cmd.h urbackupserver/apps/repair_cmd.h urbackupserver/dao/ServerCleanupDao.h urbackupserver/lmdb/lmdb.h urbackupserver/lmdb/midl.h urbackupserver/LMDBFileIndex.h urbackupserver/create_files_index.h urbackupserver/FileIndex.h urbackupserver/serverinterface/rights.h urbackupserver/server_dir_links.h urbackupserver/dao/ServerBackupDao.h urbackupserver/apps/app.h urbackupserver/apps/export_auth_log.h urbackupserver/serverinterface/login.h urbackupserver/ServerDownloadThread.h urbackupserver/ServerDownloadThreadGroup.h common/adler32.h urbackupcommon/file_metadata.h urbackupcommon/filelist_utils.h urbackupserver/Backup.h urbackupserver/ImageBackup.h urbackupserver/FileBackup.h urbackupserver/IncrFileBackup.h urbackupserver/FullFileBackup.h urbackupserver/ContinuousBackup.h urbackupserver/ThrottleUpdater.h urbackupcommon/glob.h urbackupserver/FileMetadataDownloadThread.h urbackupserver/restore_client.h urbackupcommon/chunk_hasher.h urbackupcommon/WalCheckpointThread.h urbackupcommon/CompressedPipe2.h urlplugin/IUrlFactory.h urlplugin/pluginmgr.h urlplugin/UrlFactory.h StaticPluginRegistration.h $(cryptoplugin_headers) $(fileservplugin_headers) $(fsimageplugin_headers) $(tclap_headers) urbackupserver/backup_server_db.h urbackupcommon/SparseFile.h urbackupcommon/ExtentIterator.h urbackupserver/dao/ServerLinkDao.h urbackupserver/dao/ServerLinkJournalDao.h urbackupcommon/server_compat.h urbackupserver/dao/ServerFilesDao.h urbackupserver/apps/skiphash_copy.h urbackupserver/apps/check_files_index.h urbackupserver/apps/patch.h urbackupserver/serverinterface/backups.h urbackupserver/server_continuous.h urbackupcommon/change_ids.h urbackupcommon/TreeHash.h urbackupserver/copy_storage.h urbackupserver/ImageMount.h common/bitmap.h $(cryptopp_headers) common/miniz.h urbackupserver/DataplanDb.h common/lrucache.h urbackupserver/PhashLoad.h fileservplugin/IPipeFileExt.h urbackupserver/Alerts.h urbackupserver/Mailer.h urbackupserver/alert_lua.h urbackupserver/alert_pulseway_lua.h $(luaplugin_headers) urbackupserver/LogReport.h urbackupserver/report_lua.h urbackupcommon/CompressedPipeZstd.h blockalign_src/main.cpp blockalign_src/crc32c-adler.cpp blockalign_src/crc.cpp blockalign_src/crc.h urbackupserver/WebSocketConnector.h urbackupcommon/WebSocketPipe.h $(zstd_headers) +noinst_HEADERS=SessionMgr.h WorkerThread.h Helper_win32.h Database.h defaults.h ServiceAcceptor.h Query.h SettingsReader.h file.h file_memory.h MemorySettingsReader.h Condition_lin.h LookupService.h Template.h types.h DBSettingsReader.h stringtools.h ThreadPool.h libs.h vld_.h ServiceWorker.h StreamPipe.h LoadbalancerClient.h socket_header.h FileSettingsReader.h SelectThread.h md5.h vld.h Table.h Client.h MemoryPipe.h Mutex_lin.h AcceptThread.h OutputStream.h Server.h Interface/SessionMgr.h Interface/Service.h Interface/PluginMgr.h Interface/Database.h Interface/Pipe.h Interface/CustomClient.h Interface/User.h Interface/Query.h Interface/SettingsReader.h Interface/Types.h Interface/Template.h Interface/ThreadPool.h Interface/Mutex.h Interface/File.h Interface/Condition.h Interface/Table.h Interface/Plugin.h Interface/Thread.h Interface/Action.h Interface/Object.h Interface/OutputStream.h Interface/Server.h libfastcgi/fastcgi.hpp sqlite/sqlite3.h sqlite/sqlite3ext.h utf8/utf8.h utf8/utf8/checked.h utf8/utf8/core.h utf8/utf8/unchecked.h cryptoplugin/ICryptoFactory.h cryptoplugin/IAESEncryption.h cryptoplugin/IAESDecryption.h Interface/DatabaseFactory.h Interface/DatabaseInt.h SQLiteFactory.h sqlite/shell.h PipeThrottler.h Interface/PipeThrottler.h mt19937ar.h DatabaseCursor.h Interface/DatabaseCursor.h Interface/SharedMutex.h Interface/WebSocket.h SharedMutex_lin.h httpserver/HTTPAction.h httpserver/HTTPClient.h httpserver/HTTPFile.h httpserver/HTTPProxy.h httpserver/HTTPService.h httpserver/IndexFiles.h httpserver/MIMEType.h httpserver/HTTPSocket.h urbackupserver/server_ping.h urbackupserver/server_cleanup.h urbackupcommon/os_functions.h urbackupcommon/json.h urbackupserver/serverinterface/helper.h urbackupserver/serverinterface/action_header.h urbackupserver/serverinterface/actions.h urbackupserver/server_writer.h urbackupcommon/settings.h urbackupserver/server_settings.h urbackupserver/zero_hash.h urbackupserver/server_update.h urbackupserver/server_log.h urbackupserver/server_hash.h urbackupserver/server_status.h urbackupcommon/bufmgr.h urbackupserver/server_update_stats.h urbackupcommon/sha2/sha2.h urbackupcommon/fileclient/FileClient.h common/data.h urbackupcommon/fileclient/socket_header.h urbackupcommon/fileclient/tcpstack.h urbackupcommon/fileclient/packet_ids.h urbackupserver/database.h urbackupserver/mbr_code.h urbackupserver/action_header.h urbackupcommon/escape.h urbackupserver/server.h urbackupserver/server_running.h urbackupserver/server_prepare_hash.h urbackupserver/actions.h urbackupserver/server_channel.h urbackupserver/ClientMain.h urbackupserver/treediff/TreeDiff.h urbackupserver/treediff/TreeNode.h urbackupserver/treediff/TreeReader.h fileservplugin/IFileServFactory.h fileservplugin/IFileServ.h urlplugin/IUrlFactory.h urbackupcommon/capa_bits.h cryptoplugin/ICryptoFactory.h urbackupcommon/fileclient/FileClientChunked.h urbackupserver/ChunkPatcher.h urbackupcommon/CompressedPipe.h urbackupcommon/InternetServicePipe.h urbackupcommon/InternetServicePipe2.h urbackupcommon/InternetServiceIDs.h urbackupserver/InternetServiceConnector.h md5.h urbackupcommon/settingslist.h urbackupserver/server_archive.h cryptoplugin/IZlibCompression.h cryptoplugin/IZlibDecompression.h cryptoplugin/ICryptoFactory.h cryptoplugin/IAESEncryption.h cryptoplugin/IAESDecryption.h fileservplugin/chunk_settings.h urbackupcommon/internet_pipe_capabilities.h urbackupcommon/mbrdata.h urbackupserver/filedownload.h urbackupserver/snapshot_helper.h urbackupserver/apps/cleanup_cmd.h urbackupserver/apps/repair_cmd.h urbackupserver/dao/ServerCleanupDao.h urbackupserver/lmdb/lmdb.h urbackupserver/lmdb/midl.h urbackupserver/LMDBFileIndex.h urbackupserver/create_files_index.h urbackupserver/FileIndex.h urbackupserver/serverinterface/rights.h urbackupserver/server_dir_links.h urbackupserver/dao/ServerBackupDao.h urbackupserver/apps/app.h urbackupserver/apps/export_auth_log.h urbackupserver/serverinterface/login.h urbackupserver/ServerDownloadThread.h urbackupserver/ServerDownloadThreadGroup.h common/adler32.h urbackupcommon/file_metadata.h urbackupcommon/filelist_utils.h urbackupserver/Backup.h urbackupserver/ImageBackup.h urbackupserver/FileBackup.h urbackupserver/IncrFileBackup.h urbackupserver/FullFileBackup.h urbackupserver/ContinuousBackup.h urbackupserver/ThrottleUpdater.h urbackupcommon/glob.h urbackupserver/FileMetadataDownloadThread.h urbackupserver/restore_client.h urbackupcommon/chunk_hasher.h urbackupcommon/WalCheckpointThread.h urbackupcommon/CompressedPipe2.h urlplugin/IUrlFactory.h urlplugin/pluginmgr.h urlplugin/UrlFactory.h StaticPluginRegistration.h $(cryptoplugin_headers) $(fileservplugin_headers) $(fsimageplugin_headers) $(tclap_headers) urbackupserver/backup_server_db.h urbackupcommon/SparseFile.h urbackupcommon/ExtentIterator.h urbackupserver/dao/ServerLinkDao.h urbackupserver/dao/ServerLinkJournalDao.h urbackupcommon/server_compat.h urbackupserver/dao/ServerFilesDao.h urbackupserver/apps/skiphash_copy.h urbackupserver/apps/check_files_index.h urbackupserver/apps/patch.h urbackupserver/serverinterface/backups.h urbackupserver/server_continuous.h urbackupcommon/change_ids.h urbackupcommon/TreeHash.h urbackupserver/copy_storage.h urbackupserver/ImageMount.h common/bitmap.h $(cryptopp_headers) common/miniz.h urbackupserver/DataplanDb.h common/lrucache.h urbackupserver/PhashLoad.h fileservplugin/IPipeFileExt.h urbackupserver/Alerts.h urbackupserver/Mailer.h urbackupserver/alert_lua.h urbackupserver/alert_pulseway_lua.h $(luaplugin_headers) urbackupserver/LogReport.h urbackupserver/report_lua.h urbackupcommon/CompressedPipeZstd.h blockalign_src/main.cpp blockalign_src/crc32c-adler.cpp blockalign_src/crc.cpp blockalign_src/crc.h urbackupserver/WebSocketConnector.h urbackupcommon/WebSocketPipe.h urbackupserver/serverinterface/settings.h $(zstd_headers) EXTRA_DIST=docs/urbackupsrv.1 init.d_server defaults_server logrotate_urbackupsrv urbackup-server.service urbackup-server-firewalld.xml urbackup/status.htm urbackupserver/www/js/*.js urbackupserver/www/js/vs/* urbackupserver/www/*.htm urbackupserver/www/*.ico urbackupserver/www/css/*.css urbackupserver/www/images/*.png urbackupserver/www/images/*.gif urbackupserver/www/*.ico urbackupserver/urbackup_ecdsa409k1.pub urbackupserver/www/swf/* urbackupserver/www/fonts/* tclap/COPYING tclap/AUTHORS server-license.txt urbackup/dataplan_db.txt diff --git a/MemoryPipe.cpp b/MemoryPipe.cpp index d81862d04..42baad727 100644 --- a/MemoryPipe.cpp +++ b/MemoryPipe.cpp @@ -261,4 +261,18 @@ void CMemoryPipe::resetTransferedBytes(void) bool CMemoryPipe::Flush( int timeoutms/*=-1 */ ) { return true; +} + +void CMemoryPipe::setUsageString(const std::string& str) +{ +} + +bool CMemoryPipe::setCompressionSettings(const SCompressionSettings& params) +{ + return false; +} + +bool CMemoryPipe::setOption(const SocketOption opt) +{ + return false; } diff --git a/MemoryPipe.h b/MemoryPipe.h index df540ab56..9a8d58bc6 100644 --- a/MemoryPipe.h +++ b/MemoryPipe.h @@ -28,15 +28,20 @@ class CMemoryPipe : public IPipe virtual size_t getNumElements(void); virtual size_t getNumWaiters(); - virtual void addThrottler(IPipeThrottler *throttler); - virtual void addOutgoingThrottler(IPipeThrottler *throttler); + virtual void addThrottler(IPipeThrottler *throttler); + virtual void addOutgoingThrottler(IPipeThrottler *throttler); virtual void addIncomingThrottler(IPipeThrottler *throttler); - virtual _i64 getTransferedBytes(void); + virtual _i64 getTransferedBytes(void); virtual void resetTransferedBytes(void); virtual bool Flush( int timeoutms=-1 ); + virtual void setUsageString(const std::string& str); + + virtual bool setCompressionSettings(const SCompressionSettings& params); + + bool setOption(const SocketOption opt) override; private: std::deque queue; diff --git a/OpenSSLPipe.cpp b/OpenSSLPipe.cpp index c3f8d2cb3..6d5f40b8d 100644 --- a/OpenSSLPipe.cpp +++ b/OpenSSLPipe.cpp @@ -446,31 +446,34 @@ bool OpenSSLPipe::Write(const char * buffer, size_t bsize, int timeoutms, bool f if (bsize == 0) return true; - if (!bpipe->isWritable(timeoutms)) + while(true) { - return false; - } + if (!bpipe->isWritable(timeoutms)) + { + return false; + } - int rc = BIO_write(bbio, buffer, static_cast(bsize)); + int rc = BIO_write(bbio, buffer, static_cast(bsize)); - if (rc <= 0) - { - if (!BIO_should_retry(bbio)) + if (rc <= 0) { - has_error = true; + if (!BIO_should_retry(bbio)) + { + has_error = true; + return false; + } } - return false; - } - else - { - if (rc < bsize) + else { - bpipe->doThrottle(rc, true, true); + if (rc < bsize) + { + bpipe->doThrottle(rc, true, true); - return Write(buffer + rc, bsize - rc, -1, flush); - } + return Write(buffer + rc, bsize - rc, -1, flush); + } - return true; + return true; + } } } @@ -554,4 +557,19 @@ void OpenSSLPipe::resetTransferedBytes(void) bpipe->resetTransferedBytes(); } +void OpenSSLPipe::setUsageString(const std::string& str) +{ + bpipe->setUsageString(str); +} + +bool OpenSSLPipe::setCompressionSettings(const SCompressionSettings& params) +{ + return false; +} + +bool OpenSSLPipe::setOption(const SocketOption opt) +{ + bpipe->setOption(opt); +} + #endif //WITH_OPENSSL diff --git a/OpenSSLPipe.h b/OpenSSLPipe.h index 61e150228..1005ea5b1 100644 --- a/OpenSSLPipe.h +++ b/OpenSSLPipe.h @@ -58,7 +58,12 @@ class OpenSSLPipe : public IPipe virtual _i64 getTransferedBytes(void); virtual void resetTransferedBytes(void); + + virtual void setUsageString(const std::string& str); + virtual bool setCompressionSettings(const SCompressionSettings& params); + + bool setOption(const SocketOption opt); private: std::auto_ptr bpipe; diff --git a/SChannelPipe.cpp b/SChannelPipe.cpp index d1b60d4ca..926358f22 100644 --- a/SChannelPipe.cpp +++ b/SChannelPipe.cpp @@ -12,7 +12,7 @@ SChannelPipe::SChannelPipe(CStreamPipe * bpipe) : bpipe(bpipe), has_cred_handle(false), has_ctxt_handle(false), decbuf_pos(0), sendbuf_pos(0), last_flush_time(0), - has_error(false) + has_error(false), incomplete_message(false) { } @@ -246,6 +246,21 @@ bool SChannelPipe::ssl_connect_negotiate(int timeoutms, bool do_read) return connected; } +void SChannelPipe::setUsageString(const std::string& str) +{ + bpipe->setUsageString(str); +} + +bool SChannelPipe::setCompressionSettings(const SCompressionSettings& params) +{ + return false; +} + +bool SChannelPipe::setOption(const SocketOption opt) +{ + return bpipe->setOption(opt); +} + void SChannelPipe::init() { @@ -291,12 +306,15 @@ size_t SChannelPipe::Read(char * buffer, size_t bsize, int timeoutms) encbuf.resize(encbuf.size() + bsize); } - size_t read = bpipe->Read(&encbuf[encbuf_pos], bsize, timeoutms); - - if (read == 0) - return 0; + if (encbuf_pos == 0) + { + size_t read = bpipe->Read(&encbuf[encbuf_pos], bsize, timeoutms); - encbuf_pos += read; + if (read == 0) + return 0; + + encbuf_pos += read; + } size_t orig_bsize = bsize; @@ -317,7 +335,7 @@ size_t SChannelPipe::Read(char * buffer, size_t bsize, int timeoutms) size_t read = bpipe->Read(&encbuf[encbuf_pos], encbuf_size_incr, remaining_time); if (read == 0) - return 0; + return orig_bsize - bsize; encbuf_pos += read; } @@ -337,6 +355,8 @@ size_t SChannelPipe::Read(char * buffer, size_t bsize, int timeoutms) res = sec->DecryptMessage(&ctxt_handle, &inbuf_desc, 0, NULL); + incomplete_message = res == SEC_E_INCOMPLETE_MESSAGE; + if (res == SEC_E_OK || res== SEC_I_RENEGOTIATE) { @@ -439,6 +459,8 @@ bool SChannelPipe::Flush(int timeoutms) if (has_error) return false; + last_flush_time = Server->getTimeMS(); + size_t sendbuf_off = 0; while (sendbuf_pos- sendbuf_off> 0) { @@ -516,7 +538,11 @@ bool SChannelPipe::isReadable(int timeoutms) if (has_error) return false; - return bpipe->isReadable(timeoutms); + if (decbuf_pos > 0) + return true; + + if (encbuf_pos > 0 && !incomplete_message) + return true; } bool SChannelPipe::hasError(void) diff --git a/SChannelPipe.h b/SChannelPipe.h index 09bf63c95..02a01f4eb 100644 --- a/SChannelPipe.h +++ b/SChannelPipe.h @@ -50,6 +50,12 @@ class SChannelPipe : public IPipe virtual void resetTransferedBytes(void); + virtual void setUsageString(const std::string& str); + + virtual bool setCompressionSettings(const SCompressionSettings& params); + + virtual bool setOption(const SocketOption opt); + private: bool ssl_connect_negotiate(int timeoutms, bool do_read); @@ -73,6 +79,7 @@ class SChannelPipe : public IPipe std::vector header_buf; std::vector trailer_buf; bool has_error; + bool incomplete_message; SecPkgContext_StreamSizes stream_sizes; }; diff --git a/SQLGen/SQLGen.vcxproj b/SQLGen/SQLGen.vcxproj index 96dc5501a..bf517bdfb 100644 --- a/SQLGen/SQLGen.vcxproj +++ b/SQLGen/SQLGen.vcxproj @@ -22,33 +22,34 @@ {F8E6B10F-B0A5-4DEB-BEB1-B6BC221153C0} Win32Proj SQLGen + 10.0 Application true Unicode - v140 + v142 Application true Unicode - v140 + v142 Application false true Unicode - v140 + v142 Application false true Unicode - v140 + v142 @@ -176,6 +177,7 @@ + diff --git a/SQLGen/SQLGen.vcxproj.filters b/SQLGen/SQLGen.vcxproj.filters index e9c2f88c1..7cbb7abed 100644 --- a/SQLGen/SQLGen.vcxproj.filters +++ b/SQLGen/SQLGen.vcxproj.filters @@ -135,6 +135,9 @@ Server + + Server + diff --git a/Server.cpp b/Server.cpp index d387f9253..be0e46604 100644 --- a/Server.cpp +++ b/Server.cpp @@ -222,6 +222,7 @@ void CServer::setup(void) #endif CQuery::init_mutex(); + CStreamPipe::init_mutex(); #ifdef MODE_WIN File::init_mutex(); @@ -428,24 +429,24 @@ void CServer::Log( const std::string &pStr, int LogLevel) if(log_console_time) { - std::cout << buffer; + std::cout << buffer; } if( LogLevel==LL_ERROR ) { - std::cout << "ERROR: " << pStr << std::endl; + std::cout << "ERROR: " << pStr << std::endl; if(logfile_a) logfile << buffer << "ERROR: " << pStr << std::endl; } else if( LogLevel==LL_WARNING ) { - std::cout << "WARNING: " << pStr << std::endl; + std::cout << "WARNING: " << pStr << std::endl; if(logfile_a) logfile<< buffer << "WARNING: " << pStr << std::endl; } else { - std::cout << pStr << std::endl; + std::cout << pStr << std::endl; if(logfile_a) logfile << buffer << pStr << std::endl; } @@ -1059,7 +1060,7 @@ IPipe* CServer::ConnectStream(std::string pServer, unsigned short pPort, unsigne for (size_t i = 0; i < lookup_result.size(); ++i) { - IPipe* ret = ConnectStream(lookup_result[i], pPort, pTimeoutms); + IPipe* ret = ConnectStream(pServer, lookup_result[i], pPort, pTimeoutms); if (ret != NULL) return ret; } @@ -1068,7 +1069,7 @@ IPipe* CServer::ConnectStream(std::string pServer, unsigned short pPort, unsigne } -IPipe* CServer::ConnectStream(const SLookupBlockingResult& lookup_result, unsigned short pPort, unsigned int pTimeoutms) +IPipe* CServer::ConnectStream(const std::string& connect_str, const SLookupBlockingResult& lookup_result, unsigned short pPort, unsigned int pTimeoutms) { union { @@ -1161,12 +1162,12 @@ IPipe* CServer::ConnectStream(const SLookupBlockingResult& lookup_result, unsign } else { - return new CStreamPipe(s); + return new CStreamPipe(s, "tcp " + connect_str); } #else if(rc!=SOCKET_ERROR) { - return new CStreamPipe(s); + return new CStreamPipe(s, "tcp "+connect_str); } #endif @@ -1197,8 +1198,12 @@ IPipe* CServer::ConnectStream(const SLookupBlockingResult& lookup_result, unsign rc=getsockopt(s, SOL_SOCKET, SO_ERROR, (char*)&err, &len); if(rc<0) { - closesocket(s); - Server->Log("Error getting socket status.", LL_ERROR); +#ifdef _WIN32 + Server->Log("Error getting socket status: " + convert(WSAGetLastError()), LL_ERROR); +#else + Server->Log("Error getting socket status: " + convert(errno), LL_ERROR); +#endif + closesocket(s); return NULL; } if(err) @@ -1215,7 +1220,7 @@ IPipe* CServer::ConnectStream(const SLookupBlockingResult& lookup_result, unsign if(recv_window_size>0) setsockopt(s, SOL_SOCKET, SO_RCVBUF, (char *) &recv_window_size, sizeof(recv_window_size)); #endif - return new CStreamPipe(s); + return new CStreamPipe(s, "tcp "+connect_str); } } else @@ -1250,6 +1255,8 @@ IPipe * CServer::ConnectSslStream(const std::string & pServer, unsigned short pP return NULL; } + bpipe->setUsageString("ssl " + pServer); + return ssl_pipe; #else return NULL; @@ -1258,7 +1265,7 @@ IPipe * CServer::ConnectSslStream(const std::string & pServer, unsigned short pP IPipe *CServer::PipeFromSocket(SOCKET pSocket) { - return new CStreamPipe(pSocket); + return new CStreamPipe(pSocket, "from socket"); } void CServer::DisconnectStream(IPipe *pipe) @@ -1279,6 +1286,11 @@ std::string CServer::LookupHostname(const std::string & pIp) return hostname; } +std::vector CServer::getStreamPipeList() +{ + return CStreamPipe::getPipeList(); +} + bool CServer::RegisterPluginPerThreadModel(IPluginMgr *pPluginMgr, std::string pName) { IScopedLock lock(plugin_mutex); @@ -1552,7 +1564,7 @@ bool CServer::createThread(IThread *thread, const std::string& name, CreateThrea #ifndef _LP64 //Only on 32bit architectures - pthread_attr_setstacksize(&attr, 1*1024*1024); + pthread_attr_setstacksize(&attr, 1 * 1024 * 1024); #endif #else @@ -1681,7 +1693,7 @@ void CServer::wait(unsigned int ms) #ifdef _WIN32 Sleep(ms); #else - usleep(ms*1000); + usleep(ms * 1000); #endif } @@ -2090,13 +2102,13 @@ std::vector CServer::getSecureRandomNumbers(size_t n) std::vector ret; ret.resize(n); #ifdef _WIN32 - for (size_t i = 0; i < n; ++i) + for (size_t i = 0; i(&ret[0]); - size_t bsize = sizeof(unsigned int) * n; + size_t bsize = sizeof(unsigned int)*n; secureRandomFill(buf, bsize); #endif return ret; @@ -2134,7 +2146,7 @@ void CServer::secureRandomFill(char *buf, size_t blen) std::fstream rnd_in("/dev/urandom", std::ios::in | std::ios::binary); if (!rnd_in.is_open()) { - Log("Error opening /dev/urandom for secure random number fill. Errno " + convert((int64)errno), LL_ERROR); + Log("Error opening /dev/urandom for secure random number fill. Errno "+convert((int64)errno), LL_ERROR); randomFill(buf, blen); return; } @@ -2143,8 +2155,8 @@ void CServer::secureRandomFill(char *buf, size_t blen) assert(rnd_in.gcount() == blen); - if (rnd_in.gcount() != blen - || rnd_in.fail() || rnd_in.eof()) + if(rnd_in.gcount() != blen + || rnd_in.fail() || rnd_in.eof() ) { Log("Error reading secure random numbers fill. Errno " + convert((int64)errno), LL_ERROR); abort(); diff --git a/Server.h b/Server.h index 2ef53e002..5eb73500b 100644 --- a/Server.h +++ b/Server.h @@ -146,6 +146,7 @@ class CServer : public IServer virtual void DisconnectStream(IPipe *pipe); virtual std::string LookupHostname(const std::string& pIp); + virtual std::vector getStreamPipeList(); virtual bool RegisterPluginPerThreadModel(IPluginMgr *pPluginMgr, std::string pName); virtual bool RegisterPluginThreadsafeModel(IPluginMgr *pPluginMgr, std::string pName); @@ -227,7 +228,7 @@ class CServer : public IServer void rotateLogfile(); - IPipe* ConnectStream(const SLookupBlockingResult& lookup_result, unsigned short pPort, unsigned int pTimeoutms); + IPipe* ConnectStream(const std::string& connect_str, const SLookupBlockingResult& lookup_result, unsigned short pPort, unsigned int pTimeoutms); int loglevel; diff --git a/ServiceWorker.cpp b/ServiceWorker.cpp index a6de95d08..1342cc365 100644 --- a/ServiceWorker.cpp +++ b/ServiceWorker.cpp @@ -263,7 +263,7 @@ void CServiceWorker::addNewClients(void) { for(size_t i=0;icreateClient(); nc->Init(tid, pipe, new_clients[i].second); clients.push_back( std::pair(nc, pipe) ); diff --git a/SettingsReader.cpp b/SettingsReader.cpp index e931f2f54..f6157c87d 100644 --- a/SettingsReader.cpp +++ b/SettingsReader.cpp @@ -22,11 +22,21 @@ #include #endif -std::string CSettingsReader::getValue(std::string key,std::string def) +std::string CSettingsReader::getValue(std::string key, const std::string& def) { std::string value; - bool b=getValue(key,&value); - if(b==false) + const bool b=getValue(key,&value); + if(!b) + return def; + else + return value; +} + +std::string CSettingsReader::getValue(const std::string& key, const char* def) +{ + std::string value; + const bool b = getValue(key, &value); + if (!b) return def; else return value; @@ -70,4 +80,15 @@ int64 CSettingsReader::getValue(std::string key, int64 def) return def; else return watoi64(value); -} \ No newline at end of file +} + +bool CSettingsReader::getValue(const std::string& key, const bool def) +{ + std::string value; + const bool b = getValue(key, &value); + if (!b) + return def; + + const std::string tkey = trim(value); + return tkey == "true" || tkey == "1" || tkey == "yes"; +} diff --git a/SettingsReader.h b/SettingsReader.h index a6dff1446..0de970058 100644 --- a/SettingsReader.h +++ b/SettingsReader.h @@ -9,11 +9,13 @@ class CSettingsReader : public ISettingsReader virtual bool getValue(std::string key, std::string *value)=0; - std::string getValue(std::string key,std::string def); + std::string getValue(const std::string& key, const char* def); + std::string getValue(std::string key, const std::string& def); std::string getValue(std::string key); int getValue(std::string key, int def); float getValue(std::string key, float def); int64 getValue(std::string key, int64 def); + bool getValue(const std::string& key, const bool def); }; #endif //CSETTINGSREADER_H diff --git a/StreamPipe.cpp b/StreamPipe.cpp index c4e699fe4..1aee28450 100644 --- a/StreamPipe.cpp +++ b/StreamPipe.cpp @@ -28,16 +28,26 @@ #include "Interface/PipeThrottler.h" #include "stringtools.h" -CStreamPipe::CStreamPipe( SOCKET pSocket) +std::map CStreamPipe::active_pipes; +IMutex* CStreamPipe::active_pipes_mutex = NULL; + +CStreamPipe::CStreamPipe( SOCKET pSocket, const std::string& usage_str) : transfered_bytes(0) { s=pSocket; has_error=false; + IScopedLock lock(active_pipes_mutex); + active_pipes[this] = usage_str; } CStreamPipe::~CStreamPipe() { closesocket(s); + + IScopedLock lock(active_pipes_mutex); + std::map::iterator it = active_pipes.find(this); + if (it != active_pipes.end()) + active_pipes.erase(it); } namespace @@ -349,6 +359,45 @@ bool CStreamPipe::doThrottle(size_t new_bytes, bool outgoing, bool wait) } } +void CStreamPipe::init_mutex() +{ + active_pipes_mutex = Server->createMutex(); +} + +void CStreamPipe::setUsageString(const std::string& str) +{ + IScopedLock lock(active_pipes_mutex); + active_pipes[this] = str; +} + +std::vector CStreamPipe::getPipeList() +{ + std::vector ret; + IScopedLock lock(active_pipes_mutex); + for (std::map::iterator it =active_pipes.begin(); + it!=active_pipes.end();++it) + ret.push_back(it->second); + + return ret; +} + +bool CStreamPipe::setCompressionSettings(const SCompressionSettings& params) +{ + return false; +} + +bool CStreamPipe::setOption(const SocketOption opt) +{ + switch (opt) + { + case SocketOption_NoDelay: + int flag; + flag = 1; + return setsockopt(s, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast(&flag), sizeof(int)) == 0; + } + return false; +} + _i64 CStreamPipe::getTransferedBytes(void) { return transfered_bytes; diff --git a/StreamPipe.h b/StreamPipe.h index 18bde63a7..20d7bb95c 100644 --- a/StreamPipe.h +++ b/StreamPipe.h @@ -1,13 +1,14 @@ #pragma once #include "Interface/Pipe.h" +#include "Interface/Mutex.h" #include "socket_header.h" #include class CStreamPipe : public IPipe { public: - CStreamPipe( SOCKET pSocket); + CStreamPipe( SOCKET pSocket, const std::string& usage_str); ~CStreamPipe(); virtual size_t Read(char *buffer, size_t bsize, int timeoutms); @@ -41,6 +42,16 @@ class CStreamPipe : public IPipe bool doThrottle(size_t new_bytes, bool outgoing, bool wait); + static void init_mutex(); + + virtual void setUsageString(const std::string& str); + + static std::vector getPipeList(); + + virtual bool setCompressionSettings(const SCompressionSettings& params); + + virtual bool setOption(const SocketOption opt); + private: SOCKET s; @@ -51,5 +62,6 @@ class CStreamPipe : public IPipe std::vector incoming_throttlers; std::vector outgoing_throttlers; - + static std::map active_pipes; + static IMutex* active_pipes_mutex; }; diff --git a/UrBackupBackend.sln b/UrBackupBackend.sln index 766731a3c..2ace2fcfb 100644 --- a/UrBackupBackend.sln +++ b/UrBackupBackend.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.30114.105 +# Visual Studio Version 17 +VisualStudioVersion = 17.11.35327.3 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fileservplugin", "fileservplugin\fileservplugin.vcxproj", "{B1F1AF2E-E544-45F7-864A-883461A4B574}" EndProject @@ -31,144 +31,220 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "luaplugin", "luaplugin\luap EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|ARM64 = Debug|ARM64 Debug|Win32 = Debug|Win32 Debug|x64 = Debug|x64 + Release Service|ARM64 = Release Service|ARM64 Release Service|Win32 = Release Service|Win32 Release Service|x64 = Release Service|x64 + Release|ARM64 = Release|ARM64 Release|Win32 = Release|Win32 Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution + {B1F1AF2E-E544-45F7-864A-883461A4B574}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {B1F1AF2E-E544-45F7-864A-883461A4B574}.Debug|ARM64.Build.0 = Debug|ARM64 {B1F1AF2E-E544-45F7-864A-883461A4B574}.Debug|Win32.ActiveCfg = Debug|Win32 {B1F1AF2E-E544-45F7-864A-883461A4B574}.Debug|Win32.Build.0 = Debug|Win32 {B1F1AF2E-E544-45F7-864A-883461A4B574}.Debug|x64.ActiveCfg = Debug|x64 {B1F1AF2E-E544-45F7-864A-883461A4B574}.Debug|x64.Build.0 = Debug|x64 + {B1F1AF2E-E544-45F7-864A-883461A4B574}.Release Service|ARM64.ActiveCfg = Release|ARM64 + {B1F1AF2E-E544-45F7-864A-883461A4B574}.Release Service|ARM64.Build.0 = Release|ARM64 {B1F1AF2E-E544-45F7-864A-883461A4B574}.Release Service|Win32.ActiveCfg = Release|x64 {B1F1AF2E-E544-45F7-864A-883461A4B574}.Release Service|x64.ActiveCfg = Release|x64 {B1F1AF2E-E544-45F7-864A-883461A4B574}.Release Service|x64.Build.0 = Release|x64 + {B1F1AF2E-E544-45F7-864A-883461A4B574}.Release|ARM64.ActiveCfg = Release|ARM64 + {B1F1AF2E-E544-45F7-864A-883461A4B574}.Release|ARM64.Build.0 = Release|ARM64 {B1F1AF2E-E544-45F7-864A-883461A4B574}.Release|Win32.ActiveCfg = Release|Win32 {B1F1AF2E-E544-45F7-864A-883461A4B574}.Release|Win32.Build.0 = Release|Win32 {B1F1AF2E-E544-45F7-864A-883461A4B574}.Release|x64.ActiveCfg = Release|x64 {B1F1AF2E-E544-45F7-864A-883461A4B574}.Release|x64.Build.0 = Release|x64 + {20375DC0-38DA-4254-B479-EFA8028C29B1}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {20375DC0-38DA-4254-B479-EFA8028C29B1}.Debug|ARM64.Build.0 = Debug|ARM64 {20375DC0-38DA-4254-B479-EFA8028C29B1}.Debug|Win32.ActiveCfg = Debug|Win32 {20375DC0-38DA-4254-B479-EFA8028C29B1}.Debug|Win32.Build.0 = Debug|Win32 {20375DC0-38DA-4254-B479-EFA8028C29B1}.Debug|x64.ActiveCfg = Debug|x64 {20375DC0-38DA-4254-B479-EFA8028C29B1}.Debug|x64.Build.0 = Debug|x64 + {20375DC0-38DA-4254-B479-EFA8028C29B1}.Release Service|ARM64.ActiveCfg = Release|ARM64 + {20375DC0-38DA-4254-B479-EFA8028C29B1}.Release Service|ARM64.Build.0 = Release|ARM64 {20375DC0-38DA-4254-B479-EFA8028C29B1}.Release Service|Win32.ActiveCfg = Release|x64 {20375DC0-38DA-4254-B479-EFA8028C29B1}.Release Service|x64.ActiveCfg = Release|x64 {20375DC0-38DA-4254-B479-EFA8028C29B1}.Release Service|x64.Build.0 = Release|x64 + {20375DC0-38DA-4254-B479-EFA8028C29B1}.Release|ARM64.ActiveCfg = Release|ARM64 + {20375DC0-38DA-4254-B479-EFA8028C29B1}.Release|ARM64.Build.0 = Release|ARM64 {20375DC0-38DA-4254-B479-EFA8028C29B1}.Release|Win32.ActiveCfg = Release|Win32 {20375DC0-38DA-4254-B479-EFA8028C29B1}.Release|Win32.Build.0 = Release|Win32 {20375DC0-38DA-4254-B479-EFA8028C29B1}.Release|x64.ActiveCfg = Release|x64 {20375DC0-38DA-4254-B479-EFA8028C29B1}.Release|x64.Build.0 = Release|x64 + {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Debug|ARM64.Build.0 = Debug|ARM64 {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Debug|Win32.ActiveCfg = Debug|Win32 {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Debug|Win32.Build.0 = Debug|Win32 {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Debug|x64.ActiveCfg = Debug|x64 {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Debug|x64.Build.0 = Debug|x64 + {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Release Service|ARM64.ActiveCfg = Release|ARM64 + {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Release Service|ARM64.Build.0 = Release|ARM64 {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Release Service|Win32.ActiveCfg = Release|x64 {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Release Service|x64.ActiveCfg = Release|x64 {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Release Service|x64.Build.0 = Release|x64 + {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Release|ARM64.ActiveCfg = Release|ARM64 + {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Release|ARM64.Build.0 = Release|ARM64 {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Release|Win32.ActiveCfg = Release|Win32 {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Release|Win32.Build.0 = Release|Win32 {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Release|x64.ActiveCfg = Release|x64 {3E6BBB51-77D4-4DC7-BF09-FC6F21CA04D5}.Release|x64.Build.0 = Release|x64 + {A4E2527B-4886-4163-9411-10BF66A931BE}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {A4E2527B-4886-4163-9411-10BF66A931BE}.Debug|ARM64.Build.0 = Debug|ARM64 {A4E2527B-4886-4163-9411-10BF66A931BE}.Debug|Win32.ActiveCfg = Debug|Win32 {A4E2527B-4886-4163-9411-10BF66A931BE}.Debug|Win32.Build.0 = Debug|Win32 {A4E2527B-4886-4163-9411-10BF66A931BE}.Debug|x64.ActiveCfg = Debug|x64 {A4E2527B-4886-4163-9411-10BF66A931BE}.Debug|x64.Build.0 = Debug|x64 - {A4E2527B-4886-4163-9411-10BF66A931BE}.Release Service|Win32.ActiveCfg = Release Server|x64 - {A4E2527B-4886-4163-9411-10BF66A931BE}.Release Service|x64.ActiveCfg = Release Server|x64 - {A4E2527B-4886-4163-9411-10BF66A931BE}.Release Service|x64.Build.0 = Release Server|x64 - {A4E2527B-4886-4163-9411-10BF66A931BE}.Release|Win32.ActiveCfg = Release Server|x64 - {A4E2527B-4886-4163-9411-10BF66A931BE}.Release|x64.ActiveCfg = Release Server|x64 - {A4E2527B-4886-4163-9411-10BF66A931BE}.Release|x64.Build.0 = Release Server|x64 + {A4E2527B-4886-4163-9411-10BF66A931BE}.Release Service|ARM64.ActiveCfg = Release|ARM64 + {A4E2527B-4886-4163-9411-10BF66A931BE}.Release Service|ARM64.Build.0 = Release|ARM64 + {A4E2527B-4886-4163-9411-10BF66A931BE}.Release Service|Win32.ActiveCfg = Release|x64 + {A4E2527B-4886-4163-9411-10BF66A931BE}.Release Service|x64.ActiveCfg = Release|x64 + {A4E2527B-4886-4163-9411-10BF66A931BE}.Release Service|x64.Build.0 = Release|x64 + {A4E2527B-4886-4163-9411-10BF66A931BE}.Release|ARM64.ActiveCfg = Release|ARM64 + {A4E2527B-4886-4163-9411-10BF66A931BE}.Release|ARM64.Build.0 = Release|ARM64 + {A4E2527B-4886-4163-9411-10BF66A931BE}.Release|Win32.ActiveCfg = Release|Win32 + {A4E2527B-4886-4163-9411-10BF66A931BE}.Release|Win32.Build.0 = Release|Win32 + {A4E2527B-4886-4163-9411-10BF66A931BE}.Release|x64.ActiveCfg = Release|x64 + {A4E2527B-4886-4163-9411-10BF66A931BE}.Release|x64.Build.0 = Release|x64 + {28D66E10-BF1E-45E3-B4E0-77920126531B}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {28D66E10-BF1E-45E3-B4E0-77920126531B}.Debug|ARM64.Build.0 = Debug|ARM64 {28D66E10-BF1E-45E3-B4E0-77920126531B}.Debug|Win32.ActiveCfg = Debug|Win32 {28D66E10-BF1E-45E3-B4E0-77920126531B}.Debug|Win32.Build.0 = Debug|Win32 {28D66E10-BF1E-45E3-B4E0-77920126531B}.Debug|x64.ActiveCfg = Debug|x64 {28D66E10-BF1E-45E3-B4E0-77920126531B}.Debug|x64.Build.0 = Debug|x64 + {28D66E10-BF1E-45E3-B4E0-77920126531B}.Release Service|ARM64.ActiveCfg = Release|ARM64 + {28D66E10-BF1E-45E3-B4E0-77920126531B}.Release Service|ARM64.Build.0 = Release|ARM64 {28D66E10-BF1E-45E3-B4E0-77920126531B}.Release Service|Win32.ActiveCfg = Release|Win32 {28D66E10-BF1E-45E3-B4E0-77920126531B}.Release Service|Win32.Build.0 = Release|Win32 {28D66E10-BF1E-45E3-B4E0-77920126531B}.Release Service|x64.ActiveCfg = Release|x64 {28D66E10-BF1E-45E3-B4E0-77920126531B}.Release Service|x64.Build.0 = Release|x64 + {28D66E10-BF1E-45E3-B4E0-77920126531B}.Release|ARM64.ActiveCfg = Release|ARM64 + {28D66E10-BF1E-45E3-B4E0-77920126531B}.Release|ARM64.Build.0 = Release|ARM64 {28D66E10-BF1E-45E3-B4E0-77920126531B}.Release|Win32.ActiveCfg = Release|Win32 {28D66E10-BF1E-45E3-B4E0-77920126531B}.Release|Win32.Build.0 = Release|Win32 {28D66E10-BF1E-45E3-B4E0-77920126531B}.Release|x64.ActiveCfg = Release|x64 {28D66E10-BF1E-45E3-B4E0-77920126531B}.Release|x64.Build.0 = Release|x64 + {8546D6E2-1872-418B-9766-E40F33689BE4}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {8546D6E2-1872-418B-9766-E40F33689BE4}.Debug|ARM64.Build.0 = Debug|ARM64 {8546D6E2-1872-418B-9766-E40F33689BE4}.Debug|Win32.ActiveCfg = Debug|Win32 {8546D6E2-1872-418B-9766-E40F33689BE4}.Debug|Win32.Build.0 = Debug|Win32 {8546D6E2-1872-418B-9766-E40F33689BE4}.Debug|x64.ActiveCfg = Debug|x64 {8546D6E2-1872-418B-9766-E40F33689BE4}.Debug|x64.Build.0 = Debug|x64 + {8546D6E2-1872-418B-9766-E40F33689BE4}.Release Service|ARM64.ActiveCfg = Release Service|ARM64 + {8546D6E2-1872-418B-9766-E40F33689BE4}.Release Service|ARM64.Build.0 = Release Service|ARM64 {8546D6E2-1872-418B-9766-E40F33689BE4}.Release Service|Win32.ActiveCfg = Release Service|Win32 {8546D6E2-1872-418B-9766-E40F33689BE4}.Release Service|Win32.Build.0 = Release Service|Win32 {8546D6E2-1872-418B-9766-E40F33689BE4}.Release Service|x64.ActiveCfg = Release Service|x64 {8546D6E2-1872-418B-9766-E40F33689BE4}.Release Service|x64.Build.0 = Release Service|x64 + {8546D6E2-1872-418B-9766-E40F33689BE4}.Release|ARM64.ActiveCfg = Release|ARM64 + {8546D6E2-1872-418B-9766-E40F33689BE4}.Release|ARM64.Build.0 = Release|ARM64 {8546D6E2-1872-418B-9766-E40F33689BE4}.Release|Win32.ActiveCfg = Release|Win32 {8546D6E2-1872-418B-9766-E40F33689BE4}.Release|Win32.Build.0 = Release|Win32 {8546D6E2-1872-418B-9766-E40F33689BE4}.Release|x64.ActiveCfg = Release|x64 {8546D6E2-1872-418B-9766-E40F33689BE4}.Release|x64.Build.0 = Release|x64 + {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Debug|ARM64.Build.0 = Debug|ARM64 {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Debug|Win32.ActiveCfg = Debug|Win32 {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Debug|Win32.Build.0 = Debug|Win32 {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Debug|x64.ActiveCfg = Debug|x64 {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Debug|x64.Build.0 = Debug|x64 + {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Release Service|ARM64.ActiveCfg = Release|ARM64 + {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Release Service|ARM64.Build.0 = Release|ARM64 {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Release Service|Win32.ActiveCfg = Release|x64 {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Release Service|x64.ActiveCfg = Release|x64 {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Release Service|x64.Build.0 = Release|x64 + {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Release|ARM64.ActiveCfg = Release|ARM64 + {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Release|ARM64.Build.0 = Release|ARM64 {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Release|Win32.ActiveCfg = Release|Win32 {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Release|Win32.Build.0 = Release|Win32 {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Release|x64.ActiveCfg = Release|x64 {A9B12FBF-84D8-4BB6-B4A3-DD57F06637B0}.Release|x64.Build.0 = Release|x64 + {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Debug|ARM64.Build.0 = Debug|ARM64 {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Debug|Win32.ActiveCfg = Debug|Win32 {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Debug|Win32.Build.0 = Debug|Win32 {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Debug|x64.ActiveCfg = Debug|x64 {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Debug|x64.Build.0 = Debug|x64 + {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Release Service|ARM64.ActiveCfg = Release|ARM64 + {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Release Service|ARM64.Build.0 = Release|ARM64 {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Release Service|Win32.ActiveCfg = Release|x64 {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Release Service|x64.ActiveCfg = Release|x64 {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Release Service|x64.Build.0 = Release|x64 + {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Release|ARM64.ActiveCfg = Release|ARM64 + {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Release|ARM64.Build.0 = Release|ARM64 {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Release|Win32.ActiveCfg = Release|Win32 {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Release|Win32.Build.0 = Release|Win32 {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Release|x64.ActiveCfg = Release|x64 {D1BF1BE4-1F36-4E19-8631-CB4C93B77E9B}.Release|x64.Build.0 = Release|x64 + {09263E7C-F43C-4925-B672-22C1818D8CCD}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {09263E7C-F43C-4925-B672-22C1818D8CCD}.Debug|ARM64.Build.0 = Debug|ARM64 {09263E7C-F43C-4925-B672-22C1818D8CCD}.Debug|Win32.ActiveCfg = Debug|Win32 {09263E7C-F43C-4925-B672-22C1818D8CCD}.Debug|Win32.Build.0 = Debug|Win32 {09263E7C-F43C-4925-B672-22C1818D8CCD}.Debug|x64.ActiveCfg = Debug|x64 {09263E7C-F43C-4925-B672-22C1818D8CCD}.Debug|x64.Build.0 = Debug|x64 + {09263E7C-F43C-4925-B672-22C1818D8CCD}.Release Service|ARM64.ActiveCfg = Release|ARM64 + {09263E7C-F43C-4925-B672-22C1818D8CCD}.Release Service|ARM64.Build.0 = Release|ARM64 {09263E7C-F43C-4925-B672-22C1818D8CCD}.Release Service|Win32.ActiveCfg = Release|Win32 {09263E7C-F43C-4925-B672-22C1818D8CCD}.Release Service|Win32.Build.0 = Release|Win32 {09263E7C-F43C-4925-B672-22C1818D8CCD}.Release Service|x64.ActiveCfg = Release|x64 + {09263E7C-F43C-4925-B672-22C1818D8CCD}.Release|ARM64.ActiveCfg = Release|ARM64 + {09263E7C-F43C-4925-B672-22C1818D8CCD}.Release|ARM64.Build.0 = Release|ARM64 {09263E7C-F43C-4925-B672-22C1818D8CCD}.Release|Win32.ActiveCfg = Release|Win32 {09263E7C-F43C-4925-B672-22C1818D8CCD}.Release|Win32.Build.0 = Release|Win32 {09263E7C-F43C-4925-B672-22C1818D8CCD}.Release|x64.ActiveCfg = Release|x64 {09263E7C-F43C-4925-B672-22C1818D8CCD}.Release|x64.Build.0 = Release|x64 + {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Debug|ARM64.Build.0 = Debug|ARM64 {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Debug|Win32.ActiveCfg = Debug|Win32 {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Debug|x64.ActiveCfg = Debug|x64 {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Debug|x64.Build.0 = Debug|x64 + {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Release Service|ARM64.ActiveCfg = Release|ARM64 + {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Release Service|ARM64.Build.0 = Release|ARM64 {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Release Service|Win32.ActiveCfg = Release|Win32 {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Release Service|Win32.Build.0 = Release|Win32 {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Release Service|x64.ActiveCfg = Release|x64 + {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Release|ARM64.ActiveCfg = Release|ARM64 + {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Release|ARM64.Build.0 = Release|ARM64 {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Release|Win32.ActiveCfg = Release|Win32 {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Release|Win32.Build.0 = Release|Win32 {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Release|x64.ActiveCfg = Release|x64 {DC9628DB-0FBF-4E1C-944C-6F877E185FA7}.Release|x64.Build.0 = Release|x64 + {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Debug|ARM64.Build.0 = Debug|ARM64 {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Debug|Win32.ActiveCfg = Debug|Win32 {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Debug|Win32.Build.0 = Debug|Win32 {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Debug|x64.ActiveCfg = Debug|x64 {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Debug|x64.Build.0 = Debug|x64 + {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Release Service|ARM64.ActiveCfg = Release|ARM64 + {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Release Service|ARM64.Build.0 = Release|ARM64 {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Release Service|Win32.ActiveCfg = Release|Win32 {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Release Service|Win32.Build.0 = Release|Win32 {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Release Service|x64.ActiveCfg = Release|x64 {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Release Service|x64.Build.0 = Release|x64 + {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Release|ARM64.ActiveCfg = Release|ARM64 + {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Release|ARM64.Build.0 = Release|ARM64 {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Release|Win32.ActiveCfg = Release|Win32 {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Release|Win32.Build.0 = Release|Win32 {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Release|x64.ActiveCfg = Release|x64 {C2F8110F-6103-4669-9CA5-C332DC5FE228}.Release|x64.Build.0 = Release|x64 + {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Debug|ARM64.Build.0 = Debug|ARM64 {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Debug|Win32.ActiveCfg = Debug|Win32 {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Debug|Win32.Build.0 = Debug|Win32 {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Debug|x64.ActiveCfg = Debug|x64 {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Debug|x64.Build.0 = Debug|x64 + {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Release Service|ARM64.ActiveCfg = Release|ARM64 + {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Release Service|ARM64.Build.0 = Release|ARM64 {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Release Service|Win32.ActiveCfg = Release|Win32 {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Release Service|Win32.Build.0 = Release|Win32 {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Release Service|x64.ActiveCfg = Release|x64 {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Release Service|x64.Build.0 = Release|x64 + {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Release|ARM64.ActiveCfg = Release|ARM64 + {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Release|ARM64.Build.0 = Release|ARM64 {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Release|Win32.ActiveCfg = Release|Win32 {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Release|Win32.Build.0 = Release|Win32 {249B45EB-BDD2-4F47-B2F8-3F4E3B805ACC}.Release|x64.ActiveCfg = Release|x64 diff --git a/blockalign_src/blockalign.vcxproj b/blockalign_src/blockalign.vcxproj index 5dc95fb30..be8ef78b4 100644 --- a/blockalign_src/blockalign.vcxproj +++ b/blockalign_src/blockalign.vcxproj @@ -1,10 +1,18 @@  + + Debug + ARM64 + Debug Win32 + + Release + ARM64 + Release Win32 @@ -37,26 +45,39 @@ Application true - v142 + v143 Unicode Application false - v142 + v143 true Unicode Application true - v142 + v143 + Unicode + + + Application + true + v143 Unicode Application false - v142 + v143 + true + Unicode + + + Application + false + v143 true Unicode @@ -74,9 +95,15 @@ + + + + + + true @@ -84,12 +111,36 @@ true + + true + false false + + false + + + true + + + x64-windows-static-md + + + arm64-windows-static-md + + + x64-windows-static-md + + + x64-windows-static-md + + + x86-windows-static-md + @@ -116,6 +167,19 @@ true + + + + + Level3 + Disabled + _DEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + + Level3 @@ -150,6 +214,23 @@ true + + + Level3 + + + MaxSpeed + true + true + NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + true + true + + diff --git a/blockalign_src/crc.cpp b/blockalign_src/crc.cpp index 188ef65eb..83b8f3f88 100644 --- a/blockalign_src/crc.cpp +++ b/blockalign_src/crc.cpp @@ -25,7 +25,7 @@ // Intrinsics availible in GCC 4.3 (http://gcc.gnu.org/gcc-4.3/changes.html) and // MSVC 2008 (http://msdn.microsoft.com/en-us/library/bb892950%28v=vs.90%29.aspx) // SunCC could generate SSE4 at 12.1, but the intrinsics are missing until 12.4. -#if !defined(CRYPTOPP_DISABLE_ASM) && !defined(CRYPTOPP_DISABLE_SSE4) && !defined(_M_ARM) && ((_MSC_VER >= 1500) || (defined(__SSE4_1__) && defined(__SSE4_2__))) +#if !defined(CRYPTOPP_DISABLE_ASM) && !defined(CRYPTOPP_DISABLE_SSE4) && !defined(_M_ARM) && !defined(_M_ARM64) && ((_MSC_VER >= 1500) || (defined(__SSE4_1__) && defined(__SSE4_2__))) #define CRYPTOPP_BOOL_SSE4_INTRINSICS_AVAILABLE 1 #else #define CRYPTOPP_BOOL_SSE4_INTRINSICS_AVAILABLE 0 diff --git a/blockalign_src/crc.h b/blockalign_src/crc.h index fe4edee62..556a2f542 100644 --- a/blockalign_src/crc.h +++ b/blockalign_src/crc.h @@ -7,6 +7,7 @@ #pragma once #include +#include namespace cryptopp_crc { diff --git a/build_client.bat b/build_client.bat index 320be6f6f..68eac1e64 100644 --- a/build_client.bat +++ b/build_client.bat @@ -1,5 +1,3 @@ -call update_deps.bat - call checkout_client.bat if %errorlevel% neq 0 exit /b %errorlevel% @@ -23,14 +21,10 @@ call build_revision.bat cd "%~dp0" -copy /Y "Release Server 2003\urbackupclient_server03.dll" "Release Server 2003\urbackup_server03.dll" -copy /Y "x64\Release Server 2003\urbackupclient_server03.dll" "x64\Release Server 2003\urbackup_server03.dll" copy /Y "Release\urbackupclient.dll" "Release\urbackup.dll" copy /Y "x64\Release\urbackupclient.dll" "x64\Release\urbackup.dll" -copy /Y "Release WinXP\urbackupclient_xp.dll" "Release Server 2003\urbackup_xp.dll" - FOR /F "tokens=*" %%G IN (pdb_dirs_client.txt) DO symstore add /compress /r /f "%~dp0%%G" /s "C:\symstore" /t "UrBackup Client /v "%build_revision%" /c "Release" diff --git a/build_client_backend.bat b/build_client_backend.bat index 4f8f1913a..3bead212c 100644 --- a/build_client_backend.bat +++ b/build_client_backend.bat @@ -1,17 +1,21 @@ -call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsamd64_x86.bat" +call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvarsamd64_x86.bat" -call update_deps.bat - -msbuild UrBackupBackend.sln /p:Configuration=Release /p:Platform="win32" +msbuild UrBackupBackend.sln /p:Configuration=Release /p:Platform="win32" /p:vcpkgTriplet="x86-windows-static-md" if %errorlevel% neq 0 exit /b %errorlevel% -msbuild UrBackupBackend.sln /p:Configuration=Release /p:Platform="x64" +msbuild UrBackupBackend.sln /p:Configuration=Release /p:Platform="x64" /p:vcpkgTriplet="x64-windows-static-md" +if %errorlevel% neq 0 exit /b %errorlevel% + +msbuild UrBackupBackend.sln /p:Configuration=Release /p:Platform="ARM64" /p:vcpkgTriplet="arm64-windows-static-md" +if %errorlevel% neq 0 exit /b %errorlevel% + +msbuild CompiledServer.vcxproj /p:Configuration="Release Service" /p:Platform="x64" /p:vcpkgTriplet="x64-windows-static-md" if %errorlevel% neq 0 exit /b %errorlevel% -msbuild UrBackupBackend.sln /p:Configuration="Release Service" /p:Platform="x64" +msbuild CompiledServer.vcxproj /p:Configuration="Release Service" /p:Platform="win32" /p:vcpkgTriplet="x86-windows-static-md" if %errorlevel% neq 0 exit /b %errorlevel% -msbuild UrBackupBackend.sln /p:Configuration="Release Service" /p:Platform="win32" +msbuild CompiledServer.vcxproj /p:Configuration="Release Service" /p:Platform="arm64" /p:vcpkgTriplet="arm64-windows-static-md" if %errorlevel% neq 0 exit /b %errorlevel% exit /b 0 \ No newline at end of file diff --git a/build_server.bat b/build_server.bat index c0ec79b71..7154e0355 100644 --- a/build_server.bat +++ b/build_server.bat @@ -1,6 +1,4 @@ -call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsamd64_x86.bat" - -call update_deps.bat +call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvarsamd64_x86.bat" git reset --hard python build\replace_versions.py @@ -9,34 +7,22 @@ if %errorlevel% neq 0 exit /b %errorlevel% copy /Y "%~dp0server-license.txt" "%~dp0urbackupserver_installer_win\data_common\server-license.txt" if %errorlevel% neq 0 exit /b %errorlevel% -msbuild UrBackupBackend.sln /p:Configuration=Release /p:Platform="win32" +msbuild UrBackupBackend.sln /p:Configuration=Release /p:Platform="win32" /p:vcpkgTriplet="x86-windows-static-md" if %errorlevel% neq 0 exit /b %errorlevel% -msbuild UrBackupBackend.sln /p:Configuration=Release /p:Platform="x64" -if %errorlevel% neq 0 exit /b %errorlevel% - -msbuild UrBackupBackend.sln /p:Configuration="Release Service" /p:Platform="x64" +msbuild UrBackupBackend.sln /p:Configuration=Release /p:Platform="x64" /p:vcpkgTriplet="x64-windows-static-md" if %errorlevel% neq 0 exit /b %errorlevel% -msbuild UrBackupBackend.sln /p:Configuration="Release Service" /p:Platform="win32" +msbuild CompiledServer.vcxproj /p:Configuration="Release Service" /p:Platform="x64" /p:vcpkgTriplet="x64-windows-static-md" if %errorlevel% neq 0 exit /b %errorlevel% -msbuild urbackupserver\urbackupserver.vcxproj /p:Configuration="Release Server" /p:Platform="win32" +msbuild CompiledServer.vcxproj /p:Configuration="Release Service" /p:Platform="win32" /p:vcpkgTriplet="x86-windows-static-md" if %errorlevel% neq 0 exit /b %errorlevel% -mkdir "Release Server" -copy /Y "urbackupserver\Release Server\*" "Release Server\" - -msbuild urbackupserver\urbackupserver.vcxproj /p:Configuration="Release Server" /p:Platform="x64" -if %errorlevel% neq 0 exit /b %errorlevel% - -mkdir "x64\Release Server" -copy /Y "urbackupserver\x64\Release Server\*" "x64\Release Server\" - call "%~dp0urbackupserver_installer_win/generate_msi.bat" if %errorlevel% neq 0 exit /b %errorlevel% -"C:\Program Files (x86)\NSIS\Unicode\makensis.exe" "%~dp0urbackupserver_installer_win/urbackup_server.nsi" +"C:\Program Files (x86)\NSIS\makensis.exe" "%~dp0urbackupserver_installer_win/urbackup_server.nsi" if %errorlevel% neq 0 exit /b %errorlevel% if NOT "%STORE_SYMBOLS%" == "true" GOTO skip_symbols diff --git a/build_windows_ci.bat b/build_windows_ci.bat new file mode 100644 index 000000000..908e84459 --- /dev/null +++ b/build_windows_ci.bat @@ -0,0 +1,19 @@ +call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvarsamd64_x86.bat" + +SET VCPKG_CRT_LINKAGE=dynamic +SET VCPKG_LIBRARY_LINKAGE=static + +msbuild UrBackupBackend.sln /p:Configuration=Debug /p:Platform="x64" /p:vcpkgTriplet="x64-windows-static-md" +if %errorlevel% neq 0 exit /b %errorlevel% + +msbuild UrBackupBackend.sln /p:Configuration=Release /p:Platform="win32" /p:vcpkgTriplet="x86-windows-static-md" +if %errorlevel% neq 0 exit /b %errorlevel% + +msbuild UrBackupBackend.sln /p:Configuration=Release /p:Platform="x64" /p:vcpkgTriplet="x64-windows-static-md" +if %errorlevel% neq 0 exit /b %errorlevel% + +msbuild CompiledServer.vcxproj /p:Configuration="Release Service" /p:Platform="x64" /p:vcpkgTriplet="x64-windows-static-md" +if %errorlevel% neq 0 exit /b %errorlevel% + +msbuild CompiledServer.vcxproj /p:Configuration="Release Service" /p:Platform="win32" /p:vcpkgTriplet="x86-windows-static-md" +if %errorlevel% neq 0 exit /b %errorlevel% \ No newline at end of file diff --git a/clientctl/Connector.cpp b/clientctl/Connector.cpp index 120a1f2a5..06064eef6 100644 --- a/clientctl/Connector.cpp +++ b/clientctl/Connector.cpp @@ -17,7 +17,13 @@ **************************************************************************/ #include "Connector.h" +#ifdef OS_FUNC_NO_SERVER #include "tcpstack.h" +#else +#include "../urbackupcommon/fileclient/tcpstack.h" +#include "../Interface/Server.h" +#include +#endif #include "../stringtools.h" #include "../urbackupcommon/escape.h" #include "../urbackupcommon/os_functions.h" @@ -177,7 +183,12 @@ std::string Connector::getResponse(const std::string &cmd, const std::string &ar t_args=args; CTCPStack tcpstack; - tcpstack.Send(p, cmd+"#pw="+pw+t_args); +#ifdef OS_FUNC_NO_SERVER + tcpstack.Send(p, cmd + "#pw=" + pw + t_args); +#else + std::unique_ptr pp(Server->PipeFromSocket(p)); + tcpstack.Send(pp.get(), cmd+"#pw="+pw+t_args); +#endif char *resp=NULL; char buffer[1024]; @@ -249,14 +260,13 @@ std::vector Connector::getSharedPaths(bool use_change_pw) std::string virtual_client = dir.get("virtual_client", std::string()).asString(); SBackupDir rdir = - { - dir["path"].asString(), - dir["name"].asString(), - dir["id"].asInt(), - dir["group"].asInt(), - virtual_client, - dir["flags"].asString() - }; + SBackupDir( dir["path"].asString(), + dir["name"].asString(), + dir["id"].asInt(), + dir["group"].asInt(), + virtual_client, + dir["flags"].asString(), + dir["server_default"].asInt()); ret.push_back(rdir); } @@ -270,9 +280,13 @@ std::vector Connector::getSharedPaths(bool use_change_pw) bool Connector::saveSharedPaths(const std::vector &res) { - std::string args="all_virtual_clients=1"; + std::string args="all_virtual_clients=1&enable_client_paths_use=1"; + size_t idx = 0; for (size_t i = 0; i &res) name += "/" + res[i].flags; } - args += "dir_" + convert(i) + "=" + path; - args += "&dir_" + convert(i) + "_name=" + name; - args += "&dir_" + convert(i) + "_group=" + convert(res[i].group); + args += "dir_" + convert(idx) + "=" + path; + args += "&dir_" + convert(idx) + "_name=" + name; + args += "&dir_" + convert(idx) + "_group=" + convert(res[i].group); if (!res[i].virtual_client.empty()) { - args += "&dir_" + convert(i) + "_virtual_client=" + EscapeParamString(res[i].virtual_client); + args += "&dir_" + convert(idx) + "_virtual_client=" + EscapeParamString(res[i].virtual_client); } + + ++idx; } std::string d = getResponse("SAVE BACKUP DIRS", args, true); diff --git a/clientctl/Connector.h b/clientctl/Connector.h index 7aaa828d8..54d888d91 100644 --- a/clientctl/Connector.h +++ b/clientctl/Connector.h @@ -26,12 +26,29 @@ typedef long long int int64; struct SBackupDir { + SBackupDir() + : id(0), group(0), server_default(0) + {} + + SBackupDir(std::string path, + std::string name, + int id, + int group, + std::string virtual_client, + std::string flags, + int server_default) + : path(path), name(name), id(id), + group(group), virtual_client(virtual_client), + flags(flags), server_default(server_default) + {} + std::string path; std::string name; int id; int group; std::string virtual_client; std::string flags; + int server_default; }; struct SStatus diff --git a/clientctl/clientctl.vcxproj b/clientctl/clientctl.vcxproj index 9570873a9..1fad76d43 100644 --- a/clientctl/clientctl.vcxproj +++ b/clientctl/clientctl.vcxproj @@ -1,6 +1,10 @@  + + Debug + ARM64 + Debug Win32 @@ -9,6 +13,10 @@ Debug x64 + + Release + ARM64 + Release Win32 @@ -29,27 +37,40 @@ Application true Unicode - v142 + v143 Application true Unicode - v142 + v143 + + + Application + true + Unicode + v143 Application false true Unicode - v142 + v143 Application false true Unicode - v142 + v143 + + + Application + false + true + Unicode + v143 @@ -60,12 +81,18 @@ + + + + + + true @@ -73,12 +100,33 @@ true + + true + false false + + false + + + true + + + x64-windows-static-md + + + x64-windows-static-md + + + x86-windows-static-md + + + arm64-windows-static-md + @@ -107,6 +155,20 @@ ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + Disabled + OS_FUNC_NO_SERVER;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + ws2_32.lib;%(AdditionalDependencies) + + Level3 @@ -143,6 +205,24 @@ ws2_32.lib;%(AdditionalDependencies) + + + Level3 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;OS_FUNC_NO_SERVER;%(PreprocessorDefinitions) + + + Console + true + true + true + ws2_32.lib;%(AdditionalDependencies) + + diff --git a/clientctl/main.cpp b/clientctl/main.cpp index 7ae3a344a..79f998885 100644 --- a/clientctl/main.cpp +++ b/clientctl/main.cpp @@ -1,53 +1,53 @@ -/************************************************************************* -* UrBackup - Client/Server backup system -* Copyright (C) 2011-2017 Martin Raiber -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -**************************************************************************/ - -#include -#include -#include -#include -#include -#include "Connector.h" -#include "../stringtools.h" -#include "../tclap/CmdLine.h" -#include "json/json.h" -#include "../urbackupcommon/os_functions.h" - -#ifndef _WIN32 -#include -#include -#include "../config.h" -#include -#include -#define PWFILE VARDIR "/urbackup/pw.txt" -#define PWFILE_CHANGE VARDIR "/urbackup/pw_change.txt" -#else -#include +/************************************************************************* +* UrBackup - Client/Server backup system +* Copyright (C) 2011-2017 Martin Raiber +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +**************************************************************************/ + +#include +#include +#include +#include +#include +#include "Connector.h" +#include "../stringtools.h" +#include "../tclap/CmdLine.h" +#include "json/json.h" +#include "../urbackupcommon/os_functions.h" + +#ifndef _WIN32 +#include +#include +#include "../config.h" +#include +#include +#define PWFILE VARDIR "/urbackup/pw.txt" +#define PWFILE_CHANGE VARDIR "/urbackup/pw_change.txt" +#else +#include #define PACKAGE_VERSION "$version_full_numeric$" -#define VARDIR "" -#define PWFILE "pw.txt" -#define PWFILE_CHANGE "pw_change.txt" -#endif - +#define VARDIR "" +#define PWFILE "pw.txt" +#define PWFILE_CHANGE "pw_change.txt" +#endif + #ifdef __MACH__ #include #include -#endif - +#endif + void wait(unsigned int ms) { #ifdef _WIN32 @@ -55,8 +55,8 @@ void wait(unsigned int ms) #else usleep(ms * 1000); #endif -} - +} + int64 getTimeMS() { #ifdef _WIN32 @@ -96,76 +96,76 @@ int64 getTimeMS() return static_cast(tp.tv_sec) * 1000 + tp.tv_nsec / 1000000; #endif //__APPLE__ #endif -} - -const std::string cmdline_version = PACKAGE_VERSION; - -void show_version() -{ - std::cout << "UrBackup Client Controller v" << cmdline_version << std::endl; - std::cout << "Copyright (C) 2011-2019 Martin Raiber" << std::endl; - std::cout << "This is free software; see the source for copying conditions. There is NO"<< std::endl; - std::cout << "warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."<< std::endl; -} - -void action_help(std::string cmd) -{ - std::cout << std::endl; - std::cout << "USAGE:" << std::endl; - std::cout << std::endl; - std::cout << "\t" << cmd << " [--help] [--version] []" << std::endl; - std::cout << std::endl; - std::cout << "Get specific command help with " << cmd << " --help" << std::endl; - std::cout << std::endl; - std::cout << "\t" << cmd << " start" << std::endl; - std::cout << "\t\t" "Start an incremental/full image/file backup" << std::endl; - std::cout << std::endl; - std::cout << "\t" << cmd << " status" << std::endl; - std::cout << "\t\t" "Get current backup status" << std::endl; - std::cout << std::endl; - std::cout << "\t" << cmd << " browse" << std::endl; - std::cout << "\t\t" "Browse backups and files/folders in backups" << std::endl; - std::cout << std::endl; - std::cout << "\t" << cmd << " restore-start" << std::endl; - std::cout << "\t\t" "Restore files/folders from backup" << std::endl; - std::cout << std::endl; - std::cout << "\t" << cmd << " set-settings" << std::endl; - std::cout << "\t\t" "Set backup settings" << std::endl; - std::cout << std::endl; - std::cout << "\t" << cmd << " reset-keep" << std::endl; - std::cout << "\t\t" "Reset keeping files during incremental backups" << std::endl; - std::cout << std::endl; - std::cout << "\t" << cmd << " add-backupdir" << std::endl; - std::cout << "\t\t" "Add new directory to backup set" << std::endl; - std::cout << std::endl; - std::cout << "\t" << cmd << " list-backupdirs" << std::endl; - std::cout << "\t\t" "List directories that are being backed up" << std::endl; - std::cout << std::endl; - std::cout << "\t" << cmd << " remove-backupdir" << std::endl; - std::cout << "\t\t" "Remove directory from backup set" << std::endl; - std::cout << std::endl; -} - +} + +const std::string cmdline_version = PACKAGE_VERSION; + +void show_version() +{ + std::cout << "UrBackup Client Controller v" << cmdline_version << std::endl; + std::cout << "Copyright (C) 2011-2019 Martin Raiber" << std::endl; + std::cout << "This is free software; see the source for copying conditions. There is NO"<< std::endl; + std::cout << "warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."<< std::endl; +} + +void action_help(std::string cmd) +{ + std::cout << std::endl; + std::cout << "USAGE:" << std::endl; + std::cout << std::endl; + std::cout << "\t" << cmd << " [--help] [--version] []" << std::endl; + std::cout << std::endl; + std::cout << "Get specific command help with " << cmd << " --help" << std::endl; + std::cout << std::endl; + std::cout << "\t" << cmd << " start" << std::endl; + std::cout << "\t\t" "Start an incremental/full image/file backup" << std::endl; + std::cout << std::endl; + std::cout << "\t" << cmd << " status" << std::endl; + std::cout << "\t\t" "Get current backup status" << std::endl; + std::cout << std::endl; + std::cout << "\t" << cmd << " browse" << std::endl; + std::cout << "\t\t" "Browse backups and files/folders in backups" << std::endl; + std::cout << std::endl; + std::cout << "\t" << cmd << " restore-start" << std::endl; + std::cout << "\t\t" "Restore files/folders from backup" << std::endl; + std::cout << std::endl; + std::cout << "\t" << cmd << " set-settings" << std::endl; + std::cout << "\t\t" "Set backup settings" << std::endl; + std::cout << std::endl; + std::cout << "\t" << cmd << " reset-keep" << std::endl; + std::cout << "\t\t" "Reset keeping files during incremental backups" << std::endl; + std::cout << std::endl; + std::cout << "\t" << cmd << " add-backupdir" << std::endl; + std::cout << "\t\t" "Add new directory to backup set" << std::endl; + std::cout << std::endl; + std::cout << "\t" << cmd << " list-backupdirs" << std::endl; + std::cout << "\t\t" "List directories that are being backed up" << std::endl; + std::cout << std::endl; + std::cout << "\t" << cmd << " remove-backupdir" << std::endl; + std::cout << "\t\t" "Remove directory from backup set" << std::endl; + std::cout << std::endl; +} + const size_t c_speed_size = 15; -const size_t c_max_l_length = 80; - -size_t get_terminal_width() -{ -#ifndef _WIN32 +const size_t c_max_l_length = 80; + +size_t get_terminal_width() +{ +#ifndef _WIN32 struct winsize w; - if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &w) != 0) - { - return c_max_l_length; - } - else - { - return w.ws_col; - } -#else - return c_max_l_length; -#endif -} - + if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &w) != 0) + { + return c_max_l_length; + } + else + { + return w.ws_col; + } +#else + return c_max_l_length; +#endif +} + void draw_progress(int pc_done, double speed_bpms, int64 done_bytes, int64 total_bytes, std::string details, int detail_pc) { static size_t max_line_length = 0; @@ -237,1355 +237,1503 @@ void draw_progress(int pc_done, double speed_bpms, int64 done_bytes, int64 total std::cout << toc; std::cout.flush(); -} - -typedef int(*action_fun)(std::vector args); - -class PwClientCmd -{ -public: - PwClientCmd(TCLAP::CmdLine& cmd, bool change) - : cmd(cmd), - pw_file_arg("p", "pw-file", - "Use password in file", - false, change ? PWFILE_CHANGE : PWFILE, "path", cmd), - client_arg("c", "client", - "Start backup on this client", - false, "127.0.0.1", "hostname/IP", cmd), - change(change) - { - - } - - void wait(int64 maxtimems) - { - int64 starttime = getTimeMS(); - do - { - if (FileExists(pw_file_arg.getValue())) - { - return; - } - ::wait(100); - } while (getTimeMS() - starttime < maxtimems); - } - - bool set() - { - if (change) - { - Connector::setPWFileChange(pw_file_arg.getValue()); - } - else - { - Connector::setPWFile(pw_file_arg.getValue()); - } - - Connector::setClient(client_arg.getValue()); - - if (trim(getFile(pw_file_arg.getValue())).empty()) - { - if (errno != 0) - { - perror("urbackupclientctl"); - } - std::cerr << "Cannot read backend password from " << pw_file_arg.getValue() << std::endl; - return false; - } - else - { - return true; - } - } - -private: - TCLAP::CmdLine& cmd; - - bool change; - TCLAP::ValueArg pw_file_arg; - TCLAP::ValueArg client_arg; -}; - -std::vector get_current_processes() -{ - SStatusDetails sd = Connector::getStatusDetails(); - - std::vector ret; - - for (size_t i = 0; i < sd.running_processes.size(); ++i) - { - ret.push_back(sd.running_processes[i].process_id); - } - - return ret; -} - -const std::string spinner = "|/-\\"; - -int64 wait_for_new_process(std::string type, const std::vector& current_processes) -{ - int tries = 60; - - std::string message = "Waiting for server to start backup... "; - - for (int i = 0; i < tries; ++i) - { - int tries = 20; - SStatusDetails sd = Connector::getStatusDetails(); - - while (!sd.ok && tries>0) - { - --tries; - wait(100); - sd = Connector::getStatusDetails(); - } - - if (!sd.ok) - { - return 0; - } - - for (size_t j = 0; j < sd.running_processes.size(); ++j) - { - if (sd.running_processes[j].action == type) - { - if (std::find(current_processes.begin(), current_processes.end(), sd.running_processes[j].process_id) == current_processes.end()) - { - if (i > 0) - { - std::cout << "\r" << message << "done" << std::endl; - } - return sd.running_processes[j].process_id; - } - } - } - +} + +typedef int(*action_fun)(std::vector args); + +class PwClientCmd +{ +public: + PwClientCmd(TCLAP::CmdLine& cmd, bool change) + : cmd(cmd), + pw_file_arg("p", "pw-file", + "Use password in file", + false, change ? PWFILE_CHANGE : PWFILE, "path", cmd), + client_arg("c", "client", + "Start backup on this client", + false, "127.0.0.1", "hostname/IP", cmd), + change(change) + { + + } + + void wait(int64 maxtimems) + { + int64 starttime = getTimeMS(); + do + { + if (FileExists(pw_file_arg.getValue())) + { + return; + } + ::wait(100); + } while (getTimeMS() - starttime < maxtimems); + } + + bool set() + { + if (change) + { + Connector::setPWFileChange(pw_file_arg.getValue()); + } + else + { + Connector::setPWFile(pw_file_arg.getValue()); + } + + Connector::setClient(client_arg.getValue()); + + if (trim(getFile(pw_file_arg.getValue())).empty()) + { + if (errno != 0) + { + perror("urbackupclientctl"); + } + std::cerr << "Cannot read backend password from " << pw_file_arg.getValue() << std::endl; + return false; + } + else + { + return true; + } + } + +private: + TCLAP::CmdLine& cmd; + + bool change; + TCLAP::ValueArg pw_file_arg; + TCLAP::ValueArg client_arg; +}; + +std::vector get_current_processes() +{ + SStatusDetails sd = Connector::getStatusDetails(); + + std::vector ret; + + for (size_t i = 0; i < sd.running_processes.size(); ++i) + { + ret.push_back(sd.running_processes[i].process_id); + } + + return ret; +} + +const std::string spinner = "|/-\\"; + +int64 wait_for_new_process(std::string type, const std::vector& current_processes) +{ + int tries = 60; + + std::string message = "Waiting for server to start backup... "; + + for (int i = 0; i < tries; ++i) + { + int tries = 20; + SStatusDetails sd = Connector::getStatusDetails(); + + while (!sd.ok && tries>0) + { + --tries; + wait(100); + sd = Connector::getStatusDetails(); + } + + if (!sd.ok) + { + return 0; + } + + for (size_t j = 0; j < sd.running_processes.size(); ++j) + { + if (sd.running_processes[j].action == type) + { + if (std::find(current_processes.begin(), current_processes.end(), sd.running_processes[j].process_id) == current_processes.end()) + { + if (i > 0) + { + std::cout << "\r" << message << "done" << std::endl; + } + return sd.running_processes[j].process_id; + } + } + } + std::cout << "\r" << message << spinner[i%spinner.size()]; - std::cout.flush(); - - wait(1000); - } - - std::cout << "\r" << message << "done" << std::endl; - return 0; -} - -int follow_status(bool restore, int64 process_id) -{ - bool found_once = false; - - size_t preparing_idx = 0; - size_t waiting_for_id_idx = 0; - - std::string waiting_msg; - std::string preparing_msg; - - while (true) - { - int tries = 20; - SStatusDetails status = Connector::getStatusDetails(); - - while (!status.ok && tries>0) - { - --tries; - wait(100); - status = Connector::getStatusDetails(); - } - - if (!status.ok) - { - std::cerr << "Could not get status from backend" << std::endl; - return 3; - } - - bool found = false; - - for (size_t i = 0; i < status.running_processes.size(); ++i) - { - SRunningProcess& proc = status.running_processes[i]; - if (status.running_processes[i].process_id == process_id) - { - if (!found_once && waiting_for_id_idx>0) - { - std::cout << "\r" << waiting_msg << " done" << std::endl; - } - - found_once = true; - - if (proc.percent_done < 0) - { - if (restore) - { - preparing_msg = "Preparing restore... "; - std::cout << "\r" << preparing_msg << spinner[preparing_idx%spinner.size()]; - } - else - { - preparing_msg = "Preparing... "; - std::cout << "\r" << preparing_msg << spinner[preparing_idx%spinner.size()]; - } - - std::cout.flush(); - - ++preparing_idx; - } - else - { - if (preparing_idx > 0) - { - std::cout << "\r" << preparing_msg << "done" << std::endl; - preparing_idx = 0; - } - draw_progress(proc.percent_done, proc.speed_bpms, proc.done_bytes, proc.total_bytes, proc.details, proc.detail_pc); - } - - found = true; - break; - } - } - - if (!found) - { - for (size_t i = 0; i < status.finished_processes.size(); ++i) - { - if (status.finished_processes[i].id == process_id) - { - if (status.finished_processes[i].success) - { - std::cout << std::endl; - if (restore) - { - std::cout << "Restore completed successfully." << std::endl; - } - else - { - std::cout << "Completed successfully." << std::endl; - } - return 0; - } - else - { - std::cout << std::endl; - if (restore) - { - std::cerr << "Restore failed." << std::endl; - } - else - { - std::cerr << "Failed." << std::endl; - } - return 4; - } - } - } - - if (!found_once) - { - if (restore) - { - waiting_msg = "Starting restore. Waiting for backup server... "; - std::cout << "\r" << waiting_msg << spinner[waiting_for_id_idx%spinner.size()];; - } - else - { - waiting_msg = "Waiting for process to become available... "; - std::cout << "\r" << waiting_msg << spinner[waiting_for_id_idx%spinner.size()];; - } - - std::cout.flush(); - - ++waiting_for_id_idx; - } - } - - wait(1000); - } -} - -int action_start(std::vector args) -{ - TCLAP::CmdLine cmd("Start an incremental/full image/file backup", ' ', cmdline_version); - - TCLAP::SwitchArg incr_backup("i", "incremental", "Start incremental backup"); - TCLAP::SwitchArg full_backup("f", "full", "Start full backup"); - - cmd.xorAdd(incr_backup, full_backup); - -#ifdef _WIN32 - TCLAP::SwitchArg file_backup("l", "file", "Start file backup"); - TCLAP::SwitchArg image_backup("m", "image", "Start image backup"); - - cmd.xorAdd(file_backup, image_backup); -#endif - - TCLAP::SwitchArg non_blocking_arg("b", "non-blocking", - "Do not show backup progress and block till the backup is finished but return immediately after starting it", cmd); - - TCLAP::ValueArg virtual_client_arg("v", "virtual-client", - "Virtual client name", - false, "", "client name", cmd); - - PwClientCmd pw_client_cmd(cmd, false); - - cmd.parse(args); - - if (!pw_client_cmd.set()) - { - return 3; - } - - std::vector current_processes = get_current_processes(); - - std::string type; - int rc; -#ifdef _WIN32 - if(file_backup.getValue()) - { -#endif - type = full_backup.getValue() ? "FULL" : "INCR"; - - rc = Connector::startBackup(virtual_client_arg.getValue(), full_backup.getValue()); -#ifdef _WIN32 - } - else - { - type = full_backup.getValue() ? "FULLI" : "INCRI"; - - rc = Connector::startImage(virtual_client_arg.getValue(), full_backup.getValue()); - } -#endif - - if(rc==2) - { - std::cerr << "Backup is already running" << std::endl; - return 2; - } - else if(rc==1) - { - if (non_blocking_arg.getValue()) - { - std::cout << "Backup started" << std::endl; - return 0; - } - else - { - int64 new_process = wait_for_new_process(type, current_processes); - - if (new_process == 0) - { - std::cerr << "Timeout while waiting for server to start backup" << std::endl; - return 4; - } - - return follow_status(false, new_process); - } - } - else if(rc==3) - { - std::cerr << "Error starting backup. No backup server found." << std::endl; - return 3; - } - else - { - std::cerr << "Error starting backup." << std::endl; - return 1; - } -} - -int action_status(std::vector args) -{ - TCLAP::CmdLine cmd("Get current backup status", ' ', cmdline_version); - - PwClientCmd pw_client_cmd(cmd, false); - - TCLAP::ValueArg follow_arg("f", "follow", - "Follow proccess status", - false, 0, "process id", cmd); - - cmd.parse(args); - - if (!pw_client_cmd.set()) - { - return 3; - } - - if (follow_arg.getValue() == 0) - { - std::string status = Connector::getStatusDetailsRaw(); - if (!status.empty()) - { - std::cout << status << std::endl; - return 0; - } - else - { - std::cerr << "Error getting status" << std::endl; - return 1; - } - } - else - { - return follow_status(false, follow_arg.getValue()); - } -} - -int action_browse(std::vector args) -{ - TCLAP::CmdLine cmd("Browse backups and files/folders in backups", ' ', cmdline_version); - - PwClientCmd pw_client_cmd(cmd, false); - - TCLAP::ValueArg backupid_arg("b", "backupid", - "Backupid of backup in which to browse files/folders or \"last\" for last complete backup", - false, "", "id", cmd); - - TCLAP::ValueArg path_arg("d", "path", - "Path of folder/file to which to browse", - false, "", "path", cmd); - - TCLAP::ValueArg virtual_client_arg("v", "virtual-client", - "Virtual client name", - false, "", "client name", cmd); - - cmd.parse(args); - - if (!pw_client_cmd.set()) - { - return 3; - } - - if(path_arg.getValue().empty() && !backupid_arg.isSet()) - { - Connector::EAccessError access_error; - std::string filebackups = Connector::getFileBackupsList(virtual_client_arg.getValue(), access_error); - - if(!filebackups.empty()) - { - std::cout << filebackups << std::endl; - return 0; - } - else - { - if(access_error==Connector::EAccessError_NoServer) - { - std::cerr << "Error getting file backups. No backup server found." << std::endl; - return 2; - } - else if (access_error == Connector::EAccessError_NoTokens) - { - std::cerr << "No file backup access tokens found. Did you run a file backup yet?" << std::endl; - return 3; - } - else - { - std::cerr << "Error getting file backups" << std::endl; - return 1; - } - } - } - else - { - int* pbackupid = NULL; - int backupid = 0; - if(backupid_arg.isSet()) - { - if (backupid_arg.getValue() != "last" - && convert(atoi(backupid_arg.getValue().c_str())) != backupid_arg.getValue()) - { - std::cerr << "Not a valid backupid: \"" << backupid_arg.getValue() << "\"" << std::endl; - return 3; - } - - if (backupid_arg.getValue() != "last") - { - backupid = atoi(backupid_arg.getValue().c_str()); - } - pbackupid = &backupid; - } - Connector::EAccessError access_error; - std::string filelist = Connector::getFileList(path_arg.getValue(), pbackupid, virtual_client_arg.getValue(), access_error); - - if(!filelist.empty()) - { - std::cout << filelist << std::endl; - return 0; - } - else - { - if (access_error == Connector::EAccessError_NoServer) - { - std::cerr << "Error getting file list. No backup server found." << std::endl; - return 2; - } - else if (access_error == Connector::EAccessError_NoTokens) - { - std::cerr << "No file backup access tokens found. Did you run a file backup yet?" << std::endl; - return 3; - } - else - { - std::cerr << "Error getting file list" << std::endl; - return 1; - } - } - } -} - -int wait_for_restore(std::string restore_info) -{ - Json::Value root; - Json::Reader reader; - - if (!reader.parse(restore_info, root, false)) - { - return 1; - } - - if (root.get("ok", false) == false) - { - std::cerr << "Error starting restore. Errorcode: " << root.get("err", -1).asInt() << std::endl; - return 2; - } - - int64 process_id = root["process_id"].asInt64(); - - return follow_status(true, process_id); -} - -std::string remove_ending_slash(const std::string& path) -{ - if (path.size() > 1 - && path[path.size() - 1] == os_file_sep()[0]) - { - return path.substr(0, path.size() - 1); - } - - return path; -} - -int action_start_restore(std::vector args) -{ - TCLAP::CmdLine cmd("Restore files/folders from backup", ' ', cmdline_version); - - PwClientCmd pw_client_cmd(cmd, false); - - TCLAP::ValueArg backupid_arg("b", "backupid", - "Backupid of backup from which to restore files/folders or \"last\" for last complete backup", - true, "", "id", cmd); - - TCLAP::ValueArg path_arg("d", "path", - "Path of folder/file to restore", - false, "", "path", cmd); - - TCLAP::MultiArg map_from_arg("m", "map-from", - "Map from local output path of folders/files to a different local path", - false, "path", cmd); - - TCLAP::MultiArg map_to_arg("t", "map-to", - "Map to local output path of folders/files to a different local path", - false, "path", cmd); - - TCLAP::SwitchArg no_remove_arg("n", "no-remove", - "Do not remove files/directories not in backup", cmd); - - TCLAP::SwitchArg consider_other_fs_arg("o", "consider-other-fs", - "Consider other file systems when removing files/directories not in backup", cmd); - - TCLAP::SwitchArg non_blocking_arg("l", "non-blocking", - "Do not show restore progress and block till the restore is finished but return immediately after starting it", cmd); - - TCLAP::SwitchArg no_follow_symlinks("s", "no-follow-symlinks", - "Do not follow symlinks outside of restored path during restore", cmd); - - TCLAP::ValueArg virtual_client_arg("v", "virtual-client", - "Virtual client name", - false, "", "client name", cmd); - - cmd.parse(args); - - if (map_from_arg.getValue().size() != map_to_arg.getValue().size()) - { - std::cerr << "There need to be an equal amount of -m/--map-from and -t/--map-to arguments" << std::endl; - return 2; - } - - if (!pw_client_cmd.set()) - { - return 3; - } - - if (backupid_arg.getValue() != "last" - && convert(atoi(backupid_arg.getValue().c_str())) != backupid_arg.getValue()) - { - std::cerr << "Not a valid backupid: \"" << backupid_arg.getValue() << "\"" << std::endl; - return 2; - } - - std::vector path_map; - for (size_t i = 0; i < map_from_arg.getValue().size(); ++i) - { - SPathMap new_pm; - new_pm.source = remove_ending_slash(map_from_arg.getValue()[i]); - new_pm.target = remove_ending_slash(map_to_arg.getValue()[i]); - - if (new_pm.source == os_file_sep() - && new_pm.target != os_file_sep()) - { - new_pm.target += os_file_sep(); - } - - if (new_pm.target == os_file_sep() - && new_pm.source != os_file_sep()) - { - new_pm.target = std::string(); - } - - path_map.push_back(new_pm); - } - - int backupid = 0; - if (backupid_arg.getValue() != "last") - { - backupid = atoi(backupid_arg.getValue().c_str()); - } - - Connector::EAccessError access_error; - std::string restore_info = Connector::startRestore(path_arg.getValue(), backupid, virtual_client_arg.getValue(), - path_map, access_error, !no_remove_arg.getValue(), !consider_other_fs_arg.getValue(), - !no_follow_symlinks.getValue()); - - if(!restore_info.empty()) - { - if (non_blocking_arg.getValue()) - { - std::cout << restore_info << std::endl; - return 0; - } - else - { - return wait_for_restore(restore_info); - } - } - else - { - if(access_error == Connector::EAccessError_NoServer) - { - std::cerr << "Error starting restore. No backup server found." << std::endl; - return 2; - } - else if (access_error == Connector::EAccessError_NoTokens) - { - std::cerr << "Error starting restore. No file backup access tokens found. Did you run a file backup yet?" << std::endl; - return 3; - } - else - { - std::cerr << "Error starting restore" << std::endl; - return 1; - } - } -} - -int action_set_settings(std::vector args) -{ - TCLAP::CmdLine cmd("Set backup settings", ' ', cmdline_version); - - PwClientCmd pw_client_cmd(cmd, true); - - TCLAP::MultiArg key_arg("k", "key", - "Key of the setting to set", - false, "setting key", cmd); - - TCLAP::MultiArg value_arg("v", "value", - "New value to set the setting to", - false, "setting value", cmd); - - cmd.parse(args); - - if (key_arg.getValue().size() != value_arg.getValue().size()) - { - std::cerr << "There need to be an equal amount of -k/--key and -v/--value arguments" << std::endl; - return 2; - } - - if (!pw_client_cmd.set()) - { - return 3; - } - - std::string s_settings; - for (size_t i = 0; i < key_arg.getValue().size(); ++i) - { - s_settings += key_arg.getValue()[i] + "=" + value_arg.getValue()[i] + "\n"; - } - - s_settings += "keep_old_settings=true\n"; - - bool no_perm; - bool b = Connector::updateSettings(s_settings, no_perm); - - if (!b) - { - if (no_perm) - { - std::cerr << "Error setting settings. Client is not allowed to change settings." << std::endl; - } - else - { - std::cerr << "Error setting settings." << std::endl; - } - return 1; - } - else - { - return 0; - } -} - -int action_reset_keep(std::vector args) -{ - TCLAP::CmdLine cmd("Reset keeping files during incremental backups", ' ', cmdline_version); - - PwClientCmd pw_client_cmd(cmd, true); - - TCLAP::ValueArg virtual_client_arg("v", "virtual-client", - "Virtual client name", - false, "", "client name", cmd); - - TCLAP::ValueArg backup_folder_arg("b", "backup-folder", - "Backup folder name", - false, "", "folder name", cmd); - - TCLAP::ValueArg group_arg("g", "backup-group", - "Backup group index", - false, 0, "group index", cmd); - - cmd.parse(args); - - if (!pw_client_cmd.set()) - { - return 3; - } - - std::string ret = Connector::resetKeep(virtual_client_arg.getValue(), backup_folder_arg.getValue(), group_arg.getValue()); - - if (ret == "OK") - { - return 0; - } - else if (ret == "err_virtual_client_not_found") - { - std::cerr << "Error: Virtual client not found" << std::endl; - return 4; - } - else if (ret == "err_backup_folder_not_found") - { - std::cerr << "Error: Backup folder not found" << std::endl; - return 5; - } - else - { - std::cerr << "Error: " << ret << std::endl; - return 6; - } -} - -std::string removeChars(std::string in) + std::cout.flush(); + + wait(1000); + } + + std::cout << "\r" << message << "done" << std::endl; + return 0; +} + +int follow_status(bool restore, int64 process_id) { - char illegalchars[] = { '*', ':', '/' , '\\' }; - std::string ret; - for (size_t i = 0; i0) + { + --tries; + wait(100); + status = Connector::getStatusDetails(); + } + + if (!status.ok) + { + std::cerr << "Could not get status from backend" << std::endl; + return 3; + } + bool found = false; - for (size_t j = 0; j0) + { + std::cout << "\r" << waiting_msg << " done" << std::endl; + } + + found_once = true; + + if (proc.percent_done < 0) + { + if (restore) + { + preparing_msg = "Preparing restore... "; + std::cout << "\r" << preparing_msg << spinner[preparing_idx%spinner.size()]; + } + else + { + preparing_msg = "Preparing... "; + std::cout << "\r" << preparing_msg << spinner[preparing_idx%spinner.size()]; + } + + std::cout.flush(); + + ++preparing_idx; + } + else + { + if (preparing_idx > 0) + { + std::cout << "\r" << preparing_msg << "done" << std::endl; + preparing_idx = 0; + } + draw_progress(proc.percent_done, proc.speed_bpms, proc.done_bytes, proc.total_bytes, proc.details, proc.detail_pc); + } + found = true; break; } } + if (!found) { - ret += in[i]; + for (size_t i = 0; i < status.finished_processes.size(); ++i) + { + if (status.finished_processes[i].id == process_id) + { + if (status.finished_processes[i].success) + { + std::cout << std::endl; + if (restore) + { + std::cout << "Restore completed successfully." << std::endl; + } + else + { + std::cout << "Completed successfully." << std::endl; + } + return 0; + } + else + { + std::cout << std::endl; + if (restore) + { + std::cerr << "Restore failed." << std::endl; + } + else + { + std::cerr << "Failed." << std::endl; + } + return 4; + } + } + } + + if (!found_once) + { + if (restore) + { + waiting_msg = "Starting restore. Waiting for backup server... "; + std::cout << "\r" << waiting_msg << spinner[waiting_for_id_idx%spinner.size()];; + } + else + { + waiting_msg = "Waiting for process to become available... "; + std::cout << "\r" << waiting_msg << spinner[waiting_for_id_idx%spinner.size()];; + } + + std::cout.flush(); + + ++waiting_for_id_idx; + } } + + wait(1000); } - return ret; -} - -bool findPathName(const std::vector& dirs, const std::string &pn) +} + +int action_start(std::vector args) { - for (size_t i = 0; i virtual_client_arg("v", "virtual-client", + "Virtual client name", + false, "", "client name", cmd); + + PwClientCmd pw_client_cmd(cmd, false); + + cmd.parse(args); + + if (!pw_client_cmd.set()) { - if (dirs[i].name == pn) + return 3; + } + + std::vector current_processes = get_current_processes(); + + std::string type; + int rc; +#ifdef _WIN32 + if(file_backup.getValue()) + { +#endif + type = full_backup.getValue() ? "FULL" : "INCR"; + + rc = Connector::startBackup(virtual_client_arg.getValue(), full_backup.getValue()); +#ifdef _WIN32 + } + else + { + type = full_backup.getValue() ? "FULLI" : "INCRI"; + + rc = Connector::startImage(virtual_client_arg.getValue(), full_backup.getValue()); + } +#endif + + if(rc==2) + { + std::cerr << "Backup is already running" << std::endl; + return 2; + } + else if(rc==1) + { + if (non_blocking_arg.getValue()) { - return true; + std::cout << "Backup started" << std::endl; + return 0; + } + else + { + int64 new_process = wait_for_new_process(type, current_processes); + + if (new_process == 0) + { + std::cerr << "Timeout while waiting for server to start backup" << std::endl; + return 4; + } + + return follow_status(false, new_process); } } - return false; -} - -std::string getDefaultDirname(const std::vector& dirs, const std::string &path) + else if(rc==3) + { + std::cerr << "Error starting backup. No backup server found." << std::endl; + return 3; + } + else + { + std::cerr << "Error starting backup." << std::endl; + return 1; + } +} + +int action_status(std::vector args) { - std::string dirname = removeChars(ExtractFileName(path)); + TCLAP::CmdLine cmd("Get current backup status", ' ', cmdline_version); - if (dirname.empty()) - dirname = "rootfs"; + PwClientCmd pw_client_cmd(cmd, false); - if (findPathName(dirs, dirname)) + TCLAP::ValueArg follow_arg("f", "follow", + "Follow proccess status", + false, 0, "process id", cmd); + + cmd.parse(args); + + if (!pw_client_cmd.set()) { - for (int k = 0; k<100; ++k) + return 3; + } + + if (follow_arg.getValue() == 0) + { + std::string status = Connector::getStatusDetailsRaw(); + if (!status.empty()) { - if (!findPathName(dirs, dirname + "_" + convert(k))) + std::cout << status << std::endl; + return 0; + } + else + { + std::cerr << "Error getting status" << std::endl; + return 1; + } + } + else + { + return follow_status(false, follow_arg.getValue()); + } +} + +int action_browse(std::vector args) +{ + TCLAP::CmdLine cmd("Browse backups and files/folders in backups", ' ', cmdline_version); + + PwClientCmd pw_client_cmd(cmd, false); + + TCLAP::ValueArg backupid_arg("b", "backupid", + "Backupid of backup in which to browse files/folders or \"last\" for last complete backup", + false, "", "id", cmd); + + TCLAP::ValueArg path_arg("d", "path", + "Path of folder/file to which to browse", + false, "", "path", cmd); + + TCLAP::ValueArg virtual_client_arg("v", "virtual-client", + "Virtual client name", + false, "", "client name", cmd); + + cmd.parse(args); + + if (!pw_client_cmd.set()) + { + return 3; + } + + if(path_arg.getValue().empty() && !backupid_arg.isSet()) + { + Connector::EAccessError access_error; + std::string filebackups = Connector::getFileBackupsList(virtual_client_arg.getValue(), access_error); + + if(!filebackups.empty()) + { + std::cout << filebackups << std::endl; + return 0; + } + else + { + if(access_error==Connector::EAccessError_NoServer) { - dirname = dirname + "_" + convert(k); - break; + std::cerr << "Error getting file backups. No backup server found." << std::endl; + return 2; + } + else if (access_error == Connector::EAccessError_NoTokens) + { + std::cerr << "No file backup access tokens found. Did you run a file backup yet?" << std::endl; + return 3; } + else + { + std::cerr << "Error getting file backups" << std::endl; + return 1; + } } } + else + { + int* pbackupid = NULL; + int backupid = 0; + if(backupid_arg.isSet()) + { + if (backupid_arg.getValue() != "last" + && convert(atoi(backupid_arg.getValue().c_str())) != backupid_arg.getValue()) + { + std::cerr << "Not a valid backupid: \"" << backupid_arg.getValue() << "\"" << std::endl; + return 3; + } - return dirname; -} - -int action_add_backupdir(std::vector args) -{ - TCLAP::CmdLine cmd("Add new directory to backup set", ' ', cmdline_version); - - PwClientCmd pw_client_cmd(cmd, true); - - TCLAP::ValueArg virtual_client_arg("v", "virtual-client", - "Virtual client name", - false, "", "client name", cmd); - - TCLAP::ValueArg name_arg("n", "name", - "Backup directory name", - false, "", "name", cmd); - - TCLAP::ValueArg path_arg("d", "path", - "Backup path", - true, "", "path", cmd); - - TCLAP::ValueArg group_arg("g", "backup-group", - "Backup group index", - false, 0, "group index", cmd); - - TCLAP::SwitchArg optional_arg("o", "optional", - "Do not fail backup if path does not exist", - cmd); - - TCLAP::SwitchArg no_follow_symlinks_arg("f", "no-follow-symlinks", - "Do not follow symbolic links outside of backup path", - cmd); - - TCLAP::SwitchArg symlinks_required_arg("r", "require-symlinks", - "Fail backup if symbolic link targets do not exist", - cmd); - - TCLAP::SwitchArg one_filesystem_arg("x", "one-filesystem", - "Do not cross filesystem boundary during backup", - cmd); - - TCLAP::SwitchArg require_snapshot_arg("s", "require-snapshot", - "Fail backup if snapshot of backup path cannot be created", - cmd); - - TCLAP::SwitchArg separate_hashes_arg("a", "separate-hashes", - "Do not share local hashes with other virtual clients", - cmd); - - TCLAP::SwitchArg keep_arg("k", "keep", - "Keep deleted files and directories during incremental backups. DO NOT USE", - cmd); - - cmd.parse(args); - - if (!pw_client_cmd.set()) - { - return 3; - } - - std::string flags; - - if (optional_arg.getValue()) - { - if (!flags.empty()) flags += ","; - flags += "optional"; - } - - if (!no_follow_symlinks_arg.getValue()) - { - if (!flags.empty()) flags += ","; - flags += "follow_symlinks"; - } - - if (!symlinks_required_arg.getValue()) - { - if (!flags.empty()) flags += ","; - flags += "symlinks_optional"; - } - - if (one_filesystem_arg.getValue()) - { - if (!flags.empty()) flags += ","; - flags += "one_filesystem"; - } - - if (require_snapshot_arg.getValue()) - { - if (!flags.empty()) flags += ","; - flags += "require_snapshot"; - } - - if (!separate_hashes_arg.getValue()) - { - if (!flags.empty()) flags += ","; - flags += "share_hashes"; - } - - if (keep_arg.getValue()) - { - if (!flags.empty()) flags += ","; - flags += "keep"; - } - - std::vector backup_dirs = Connector::getSharedPaths(true); - - if (Connector::hasError()) - { - std::cerr << "Error retrieving current backup directories from backend" << std::endl; - return 1; - } - - SBackupDir new_dir; - new_dir.path = path_arg.getValue(); - if (name_arg.getValue().empty()) - { - new_dir.name = getDefaultDirname(backup_dirs, new_dir.path); - } - else - { - new_dir.name = name_arg.getValue(); - } - - new_dir.group = group_arg.getValue(); - - new_dir.flags = flags; - - new_dir.virtual_client = virtual_client_arg.getValue(); - - backup_dirs.push_back(new_dir); - - if (!Connector::saveSharedPaths(backup_dirs)) - { - std::cerr << "Error adding new backup path via backend" << std::endl; - return 2; - } - else - { - return 0; - } -} - -void ouput_val(std::string val, size_t max_size) -{ - if (val.size() > max_size) - { - val = val.substr(0, max_size); - } - std::cout << val; - for (size_t i = val.size(); i < max_size; ++i) - { - std::cout << ' '; - } -} - -void display_table(const std::vector >& rows) -{ - if (rows.empty()) - { - return; - } - - const size_t val_gap = 1; - - std::vector max_size; - max_size.resize(rows[0].size()); - - for (size_t i = 0; i < rows[0].size(); ++i) - { - for (size_t j = 0; j < rows.size(); ++j) - { - max_size[i] = (std::max)(rows[j][i].size(), max_size[i]); - } - } - - for (size_t i = 0; i < rows[0].size(); ++i) - { - ouput_val(rows[0][i], max_size[i]+ val_gap); - } - - std::cout << std::endl; - - for (size_t i = 0; i < rows[0].size(); ++i) - { - std::cout << std::string(max_size[i], '-'); - std::cout << std::string(val_gap, ' '); - } - - std::cout << std::endl; - - for (size_t i = 1; i < rows.size(); ++i) - { - for (size_t j = 0; j < rows[i].size(); ++j) - { - ouput_val(rows[i][j], max_size[j] + val_gap); - } - std::cout << std::endl; - } -} - -int action_list_backupdirs(std::vector args) -{ - TCLAP::CmdLine cmd("List directories that are being backed up", ' ', cmdline_version); - - PwClientCmd pw_client_cmd(cmd, false); - - TCLAP::ValueArg virtual_client_arg("v", "virtual-client", - "Display only for virtual with name", - false, "", "client name", cmd); - - TCLAP::SwitchArg raw_arg("r", "raw", - "Return raw JSON output", cmd); - - cmd.parse(args); - - if (!pw_client_cmd.set()) - { - return 3; - } - - if (raw_arg.getValue()) - { - std::string ret = Connector::getSharedPathsRaw(); - if (ret.empty()) - { - std::cerr << "Error retrieving current backup directories from backend" << std::endl; - return 1; - } - std::cout << ret; - std::cout.flush(); - return 0; - } - - std::vector backup_dirs = Connector::getSharedPaths(false); - - if (Connector::hasError()) - { - std::cerr << "Error retrieving current backup directories from backend" << std::endl; - return 1; - } - - if (backup_dirs.empty()) - { - std::cout << "No directories are being backed up" << std::endl; - return 0; - } - - bool has_virtual_client = false; - bool has_group = false; - - for (size_t i = 0; i < backup_dirs.size(); ++i) - { - if (!backup_dirs[i].virtual_client.empty()) - { - has_virtual_client = true; - } - if (backup_dirs[i].group != 0) - { - has_group = true; - } - } - - std::vector > tab; - - std::vector tab_header; - tab_header.push_back("PATH"); - tab_header.push_back("NAME"); - if (has_group) - { - tab_header.push_back("GROUP"); - } - if (has_virtual_client) - { - tab_header.push_back("VIRTUAL CLIENT"); - } - tab_header.push_back("FLAGS"); - - tab.push_back(tab_header); - - for (size_t i = 0; i < backup_dirs.size(); ++i) - { - std::vector row; - row.push_back(backup_dirs[i].path); - - if (backup_dirs[i].name.empty()) - { - backup_dirs[i].name = getDefaultDirname(backup_dirs, backup_dirs[i].path); - } - - row.push_back(backup_dirs[i].name); - - if (has_group) - { - row.push_back(convert(backup_dirs[i].group)); - } - - if (has_virtual_client) - { - if (backup_dirs[i].virtual_client.empty()) - { - row.push_back("-"); - } - else - { - row.push_back(backup_dirs[i].virtual_client); - } - } - - row.push_back(backup_dirs[i].flags); - - tab.push_back(row); - } - - display_table(tab); - - return 0; -} - -int action_remove_backupdir(std::vector args) -{ - TCLAP::CmdLine cmd("Remove directory from backup set", ' ', cmdline_version); - - PwClientCmd pw_client_cmd(cmd, true); - - TCLAP::ValueArg name_arg("n", "name", - "Backup directory name", - false, "", "name"); - - TCLAP::ValueArg path_arg("d", "path", - "Backup path", - true, "", "path"); - - cmd.xorAdd(name_arg, path_arg); - - cmd.parse(args); - - if (!pw_client_cmd.set()) - { - return 3; - } - - std::vector backup_dirs = Connector::getSharedPaths(true); - - if (Connector::hasError()) - { - std::cerr << "Error retrieving current backup directories from backend" << std::endl; - return 1; - } - - bool del_ok = false; - - for (size_t i = 0; i < backup_dirs.size();) - { - if (!name_arg.getValue().empty() - && backup_dirs[i].name == name_arg.getValue()) - { - backup_dirs.erase(backup_dirs.begin() + i); - del_ok = true; - } - else if (!path_arg.getValue().empty() - && backup_dirs[i].path == path_arg.getValue()) - { - backup_dirs.erase(backup_dirs.begin() + i); - del_ok = true; - } - else - { - ++i; - } - } - - if (!del_ok) - { - std::cerr << "Backup directory to remove not found" << std::endl; - return 1; - } - - if (!Connector::saveSharedPaths(backup_dirs)) - { - std::cerr << "Error removing backup directory via backend" << std::endl; - return 2; - } - else - { - return 0; - } -} - -int action_wait_for_backend(std::vector args) -{ - TCLAP::CmdLine cmd("Wait for backend to become available", ' ', cmdline_version); - - PwClientCmd pw_client_cmd(cmd, false); - - TCLAP::ValueArg time_arg("t", "time", - "Max time in seconds to wait", - false, 60, "seconds", cmd); - - cmd.parse(args); - - pw_client_cmd.wait(time_arg.getValue() * 1000); - - if (!pw_client_cmd.set()) - { - return 3; - } - - int64 starttime = getTimeMS(); - do - { - int64 thistime = getTimeMS(); - std::string d = Connector::getStatusRawNoWait(); - if (!Connector::hasError() - && !d.empty()) - { - return 0; - } - if (getTimeMS() - thistime < 30) - { - wait(100); - } - } while (getTimeMS() - starttime < time_arg.getValue() * 1000); - - std::cerr << "Could not connect to backend in specified time" << std::endl; - return 1; -} - -int main(int argc, char *argv[]) -{ - if(argc==0) - { - std::cerr << "Not enough arguments (zero arguments) -- no program name" << std::endl; - return 1; - } - -#ifdef _WIN32 - HMODULE hModule = GetModuleHandleW(NULL); - if (hModule != INVALID_HANDLE_VALUE) + if (backupid_arg.getValue() != "last") + { + backupid = atoi(backupid_arg.getValue().c_str()); + } + pbackupid = &backupid; + } + Connector::EAccessError access_error; + std::string filelist = Connector::getFileList(path_arg.getValue(), pbackupid, virtual_client_arg.getValue(), access_error); + + if(!filelist.empty()) + { + std::cout << filelist << std::endl; + return 0; + } + else + { + if (access_error == Connector::EAccessError_NoServer) + { + std::cerr << "Error getting file list. No backup server found." << std::endl; + return 2; + } + else if (access_error == Connector::EAccessError_NoTokens) + { + std::cerr << "No file backup access tokens found. Did you run a file backup yet?" << std::endl; + return 3; + } + else + { + std::cerr << "Error getting file list" << std::endl; + return 1; + } + } + } +} + +int wait_for_restore(std::string restore_info) +{ + Json::Value root; + Json::Reader reader; + + if (!reader.parse(restore_info, root, false)) { - WCHAR path[MAX_PATH]; - if (GetModuleFileNameW(hModule, path, MAX_PATH) != 0) - { - SetCurrentDirectoryW(path); - } - } -#endif - - std::vector actions; - std::vector action_funs; - actions.push_back("start"); - action_funs.push_back(action_start); - actions.push_back("status"); - action_funs.push_back(action_status); - actions.push_back("browse"); - action_funs.push_back(action_browse); - actions.push_back("restore-start"); - action_funs.push_back(action_start_restore); - actions.push_back("set-settings"); - action_funs.push_back(action_set_settings); - actions.push_back("reset-keep"); - action_funs.push_back(action_reset_keep); - actions.push_back("add-backupdir"); - action_funs.push_back(action_add_backupdir); - actions.push_back("list-backupdirs"); - action_funs.push_back(action_list_backupdirs); - actions.push_back("remove-backupdir"); - action_funs.push_back(action_remove_backupdir); - actions.push_back("wait-for-backend"); - action_funs.push_back(action_wait_for_backend); - - bool has_help=false; - bool has_version=false; - size_t action_idx=std::string::npos; - std::vector args; - args.push_back(argv[0]); - for(int i=1;i 1 + && path[path.size() - 1] == os_file_sep()[0]) + { + return path.substr(0, path.size() - 1); + } + + return path; +} + +int action_start_restore(std::vector args) +{ + TCLAP::CmdLine cmd("Restore files/folders from backup", ' ', cmdline_version); + + PwClientCmd pw_client_cmd(cmd, false); + + TCLAP::ValueArg backupid_arg("b", "backupid", + "Backupid of backup from which to restore files/folders or \"last\" for last complete backup", + true, "", "id", cmd); + + TCLAP::ValueArg path_arg("d", "path", + "Path of folder/file to restore", + false, "", "path", cmd); + + TCLAP::MultiArg map_from_arg("m", "map-from", + "Map from local output path of folders/files to a different local path", + false, "path", cmd); + + TCLAP::MultiArg map_to_arg("t", "map-to", + "Map to local output path of folders/files to a different local path", + false, "path", cmd); + + TCLAP::SwitchArg no_remove_arg("n", "no-remove", + "Do not remove files/directories not in backup", cmd); + + TCLAP::SwitchArg consider_other_fs_arg("o", "consider-other-fs", + "Consider other file systems when removing files/directories not in backup", cmd); + + TCLAP::SwitchArg non_blocking_arg("l", "non-blocking", + "Do not show restore progress and block till the restore is finished but return immediately after starting it", cmd); + + TCLAP::SwitchArg follow_symlinks("f", "follow-symlinks", + "Follow symlinks outside of restored path during restore", cmd); + + TCLAP::ValueArg virtual_client_arg("v", "virtual-client", + "Virtual client name", + false, "", "client name", cmd); + + cmd.parse(args); + + if (map_from_arg.getValue().size() != map_to_arg.getValue().size()) + { + std::cerr << "There need to be an equal amount of -m/--map-from and -t/--map-to arguments" << std::endl; + return 2; + } + + if (!pw_client_cmd.set()) + { + return 3; + } + + if (backupid_arg.getValue() != "last" + && convert(atoi(backupid_arg.getValue().c_str())) != backupid_arg.getValue()) + { + std::cerr << "Not a valid backupid: \"" << backupid_arg.getValue() << "\"" << std::endl; + return 2; + } + + std::vector path_map; + for (size_t i = 0; i < map_from_arg.getValue().size(); ++i) + { + SPathMap new_pm; + new_pm.source = remove_ending_slash(map_from_arg.getValue()[i]); + new_pm.target = remove_ending_slash(map_to_arg.getValue()[i]); + + if (new_pm.source == os_file_sep() + && new_pm.target != os_file_sep()) + { + new_pm.target += os_file_sep(); + } + + if (new_pm.target == os_file_sep() + && new_pm.source != os_file_sep()) + { + new_pm.target = std::string(); + } + + path_map.push_back(new_pm); + } + + int backupid = 0; + if (backupid_arg.getValue() != "last") + { + backupid = atoi(backupid_arg.getValue().c_str()); + } + + Connector::EAccessError access_error; + std::string restore_info = Connector::startRestore(path_arg.getValue(), backupid, virtual_client_arg.getValue(), + path_map, access_error, !no_remove_arg.getValue(), !consider_other_fs_arg.getValue(), + follow_symlinks.getValue()); + + if(!restore_info.empty()) + { + if (non_blocking_arg.getValue()) + { + std::cout << restore_info << std::endl; + return 0; + } + else + { + return wait_for_restore(restore_info); + } + } + else + { + if(access_error == Connector::EAccessError_NoServer) + { + std::cerr << "Error starting restore. No backup server found." << std::endl; + return 2; + } + else if (access_error == Connector::EAccessError_NoTokens) + { + std::cerr << "Error starting restore. No file backup access tokens found. Did you run a file backup yet?" << std::endl; + return 3; + } + else + { + std::cerr << "Error starting restore" << std::endl; + return 1; + } + } +} + +int action_set_settings(std::vector args) +{ + TCLAP::CmdLine cmd("Set backup settings", ' ', cmdline_version); + + PwClientCmd pw_client_cmd(cmd, true); + + TCLAP::MultiArg key_arg("k", "key", + "Key of the setting to set", + false, "setting key", cmd); + + TCLAP::MultiArg value_arg("v", "value", + "New value to set the setting to", + false, "setting value", cmd); + + TCLAP::SwitchArg no_merge_arg("n", "no-merge", + "Don't merge server and client settings if possible", cmd); + + TCLAP::ValueArg server_url_arg("", "server-url", + "URL of server to connect to", + false, "", "url", cmd); + + TCLAP::ValueArg name_arg("", "name", + "Client name", + false, "", "string", cmd); + + TCLAP::ValueArg authkey_arg("", "authkey", + "Server authentication key for client", + false, "", "string", cmd); + + TCLAP::ValueArg proxy_arg("", "proxy", + "HTTP CONNECT proxy to use to connect to server", + false, "", "url", cmd); + + cmd.parse(args); + + if (key_arg.getValue().size() != value_arg.getValue().size()) + { + std::cerr << "There need to be an equal amount of -k/--key and -v/--value arguments" << std::endl; + return 2; + } + + if (!pw_client_cmd.set()) + { + return 3; + } + + str_map arg_settings; + + if (server_url_arg.isSet()) + { + std::vector server_urls; + Tokenize(server_url_arg.getValue(), server_urls, ";"); + std::string internet_server; + std::string internet_server_port; + for (size_t i = 0; i < server_urls.size(); ++i) + { + std::string server_url = server_urls[i]; + std::string server_port = "55415"; + + if (server_url.find("urbackup://") != 0 && + server_url.find("wss://") != 0 && + server_url.find("ws://") != 0) + { + std::cerr << "Server URL must start with urbackup://, wss:// or ws://" << std::endl; + return 4; + } + + if (server_url.find("urbackup://") == 0) + { + std::string hostname = server_url.substr(11); + if (hostname.find(":") != std::string::npos) + { + server_port = getafter(":", server_url); + } + server_url = hostname; + } + + if (!internet_server.empty()) + internet_server += ";"; + if (!internet_server_port.empty()) + internet_server_port += ";"; + + internet_server += server_url; + internet_server_port += server_port; + } + + arg_settings["internet_server_port"] = internet_server_port; + arg_settings["internet_server"] = internet_server; + arg_settings["internet_mode_enabled"] = "true"; + } + + if (authkey_arg.isSet()) + { + arg_settings["internet_authkey"] = authkey_arg.getValue(); + arg_settings["internet_mode_enabled"] = "true"; + } + + if (name_arg.isSet()) + { + arg_settings["computername"] = name_arg.getValue(); + } + + if (proxy_arg.isSet()) + { + arg_settings["internet_server_proxy"] = proxy_arg.getValue(); + arg_settings["internet_mode_enabled"] = "true"; + } + + std::string s_settings; + for (size_t i = 0; i < key_arg.getValue().size(); ++i) + { + std::string key = key_arg.getValue()[i]; + if(arg_settings.find(key)==arg_settings.end()) + s_settings += key + "=" + value_arg.getValue()[i] + "\n"; + } + + for (str_map::const_iterator it = arg_settings.begin(); + it != arg_settings.end(); ++it) + { + s_settings += it->first + "=" + it->second + "\n"; + } + + s_settings += "set_client_settings=1\n"; + + if (!no_merge_arg.getValue()) + { + s_settings += "merge_client_settings=0\n"; + } + + bool no_perm; + bool b = Connector::updateSettings(s_settings, no_perm); + + if (!b) + { + if (no_perm) + { + std::cerr << "Error setting settings. Client is not allowed to change settings." << std::endl; + } + else + { + std::cerr << "Error setting settings." << std::endl; + } + return 1; + } + else + { + return 0; + } +} + +int action_reset_keep(std::vector args) +{ + TCLAP::CmdLine cmd("Reset keeping files during incremental backups", ' ', cmdline_version); + + PwClientCmd pw_client_cmd(cmd, true); + + TCLAP::ValueArg virtual_client_arg("v", "virtual-client", + "Virtual client name", + false, "", "client name", cmd); + + TCLAP::ValueArg backup_folder_arg("b", "backup-folder", + "Backup folder name", + false, "", "folder name", cmd); + + TCLAP::ValueArg group_arg("g", "backup-group", + "Backup group index", + false, 0, "group index", cmd); + + cmd.parse(args); + + if (!pw_client_cmd.set()) + { + return 3; + } + + std::string ret = Connector::resetKeep(virtual_client_arg.getValue(), backup_folder_arg.getValue(), group_arg.getValue()); + + if (ret == "OK") + { + return 0; + } + else if (ret == "err_virtual_client_not_found") + { + std::cerr << "Error: Virtual client not found" << std::endl; + return 4; + } + else if (ret == "err_backup_folder_not_found") + { + std::cerr << "Error: Backup folder not found" << std::endl; + return 5; + } + else + { + std::cerr << "Error: " << ret << std::endl; + return 6; + } +} + +std::string removeChars(std::string in) +{ + char illegalchars[] = { '*', ':', '/' , '\\' }; + std::string ret; + for (size_t i = 0; i& dirs, const std::string &pn) +{ + for (size_t i = 0; i& dirs, const std::string &path) +{ + std::string dirname = removeChars(ExtractFileName(path)); + + if (dirname.empty()) + dirname = "rootfs"; + + if (findPathName(dirs, dirname)) + { + for (int k = 0; k<100; ++k) + { + if (!findPathName(dirs, dirname + "_" + convert(k))) + { + dirname = dirname + "_" + convert(k); + break; + } + } + } + + return dirname; +} + +int action_add_backupdir(std::vector args) +{ + TCLAP::CmdLine cmd("Add new directory to backup set", ' ', cmdline_version); + + PwClientCmd pw_client_cmd(cmd, true); + + TCLAP::ValueArg virtual_client_arg("v", "virtual-client", + "Virtual client name", + false, "", "client name", cmd); + + TCLAP::ValueArg name_arg("n", "name", + "Backup directory name", + false, "", "name", cmd); + + TCLAP::ValueArg path_arg("d", "path", + "Backup path", + true, "", "path", cmd); + + TCLAP::ValueArg group_arg("g", "backup-group", + "Backup group index", + false, 0, "group index", cmd); + + TCLAP::SwitchArg optional_arg("o", "optional", + "Do not fail backup if path does not exist", + cmd); + + TCLAP::SwitchArg no_follow_symlinks_arg("f", "no-follow-symlinks", + "Do not follow symbolic links outside of backup path", + cmd); + + TCLAP::SwitchArg symlinks_required_arg("r", "require-symlinks", + "Fail backup if symbolic link targets do not exist", + cmd); + + TCLAP::SwitchArg one_filesystem_arg("x", "one-filesystem", + "Do not cross filesystem boundary during backup", + cmd); + + TCLAP::SwitchArg require_snapshot_arg("s", "require-snapshot", + "Fail backup if snapshot of backup path cannot be created", + cmd); + + TCLAP::SwitchArg separate_hashes_arg("a", "separate-hashes", + "Do not share local hashes with other virtual clients", + cmd); + + TCLAP::SwitchArg keep_arg("k", "keep", + "Keep deleted files and directories during incremental backups. DO NOT USE", + cmd); + + cmd.parse(args); + + if (!pw_client_cmd.set()) + { + return 3; + } + + std::string flags; + + if (optional_arg.getValue()) + { + if (!flags.empty()) flags += ","; + flags += "optional"; + } + + if (!no_follow_symlinks_arg.getValue()) + { + if (!flags.empty()) flags += ","; + flags += "follow_symlinks"; + } + + if (!symlinks_required_arg.getValue()) + { + if (!flags.empty()) flags += ","; + flags += "symlinks_optional"; + } + + if (one_filesystem_arg.getValue()) + { + if (!flags.empty()) flags += ","; + flags += "one_filesystem"; + } + + if (require_snapshot_arg.getValue()) + { + if (!flags.empty()) flags += ","; + flags += "require_snapshot"; + } + + if (!separate_hashes_arg.getValue()) + { + if (!flags.empty()) flags += ","; + flags += "share_hashes"; + } + + if (keep_arg.getValue()) + { + if (!flags.empty()) flags += ","; + flags += "keep"; + } + + std::vector backup_dirs = Connector::getSharedPaths(true); + + if (Connector::hasError()) + { + std::cerr << "Error retrieving current backup directories from backend" << std::endl; + return 1; + } + + SBackupDir new_dir; + new_dir.path = path_arg.getValue(); + + if (!os_path_absolute(new_dir.path)) + { + new_dir.path = os_get_final_path(new_dir.path); + } + + if (new_dir.path.size()>1 && new_dir.path[new_dir.path.size() - 1] == os_file_sep()[0]) + { + new_dir.path.erase(new_dir.path.size() - 1, 1); + } + + if (name_arg.getValue().empty()) + { + new_dir.name = getDefaultDirname(backup_dirs, new_dir.path); + } + else + { + new_dir.name = name_arg.getValue(); + } + + new_dir.group = group_arg.getValue(); + + new_dir.flags = flags; + + new_dir.virtual_client = virtual_client_arg.getValue(); + + backup_dirs.push_back(new_dir); + + if (!Connector::saveSharedPaths(backup_dirs)) + { + std::cerr << "Error adding new backup path via backend" << std::endl; + return 2; + } + else + { + return 0; + } +} + +void ouput_val(std::string val, size_t max_size) +{ + if (val.size() > max_size) + { + val = val.substr(0, max_size); + } + std::cout << val; + for (size_t i = val.size(); i < max_size; ++i) + { + std::cout << ' '; + } +} + +void display_table(const std::vector >& rows) +{ + if (rows.empty()) + { + return; + } + + const size_t val_gap = 1; + + std::vector max_size; + max_size.resize(rows[0].size()); + + for (size_t i = 0; i < rows[0].size(); ++i) + { + for (size_t j = 0; j < rows.size(); ++j) + { + max_size[i] = (std::max)(rows[j][i].size(), max_size[i]); + } + } + + for (size_t i = 0; i < rows[0].size(); ++i) + { + ouput_val(rows[0][i], max_size[i]+ val_gap); + } + + std::cout << std::endl; + + for (size_t i = 0; i < rows[0].size(); ++i) + { + std::cout << std::string(max_size[i], '-'); + std::cout << std::string(val_gap, ' '); + } + + std::cout << std::endl; + + for (size_t i = 1; i < rows.size(); ++i) + { + for (size_t j = 0; j < rows[i].size(); ++j) + { + ouput_val(rows[i][j], max_size[j] + val_gap); + } + std::cout << std::endl; + } +} + +int action_list_backupdirs(std::vector args) +{ + TCLAP::CmdLine cmd("List directories that are being backed up", ' ', cmdline_version); + + PwClientCmd pw_client_cmd(cmd, false); + + TCLAP::ValueArg virtual_client_arg("v", "virtual-client", + "Display only for virtual with name", + false, "", "client name", cmd); + + TCLAP::SwitchArg raw_arg("r", "raw", + "Return raw JSON output", cmd); + + cmd.parse(args); + + if (!pw_client_cmd.set()) + { + return 3; + } + + if (raw_arg.getValue()) + { + std::string ret = Connector::getSharedPathsRaw(); + if (ret.empty()) + { + std::cerr << "Error retrieving current backup directories from backend" << std::endl; + return 1; + } + std::cout << ret; + std::cout.flush(); + return 0; + } + + std::vector backup_dirs = Connector::getSharedPaths(false); + + if (Connector::hasError()) + { + std::cerr << "Error retrieving current backup directories from backend" << std::endl; + return 1; + } + + if (backup_dirs.empty()) + { + std::cout << "No directories are being backed up" << std::endl; + return 0; + } + + bool has_virtual_client = false; + bool has_group = false; + bool has_server_default = false; + + for (size_t i = 0; i < backup_dirs.size(); ++i) + { + if (!backup_dirs[i].virtual_client.empty()) + { + has_virtual_client = true; + } + if (backup_dirs[i].group != 0) + { + has_group = true; + } + if (backup_dirs[i].server_default) + has_server_default = true; + } + + std::vector > tab; + + std::vector tab_header; + tab_header.push_back("PATH"); + tab_header.push_back("NAME"); + if (has_group) + { + tab_header.push_back("GROUP"); + } + if (has_virtual_client) + { + tab_header.push_back("VIRTUAL CLIENT"); + } + tab_header.push_back("FLAGS"); + if (has_server_default) + { + tab_header.push_back("CONFIGURED ON SERVER"); + } + + tab.push_back(tab_header); + + for (size_t i = 0; i < backup_dirs.size(); ++i) + { + std::vector row; + row.push_back(backup_dirs[i].path); + + if (backup_dirs[i].name.empty()) + { + backup_dirs[i].name = getDefaultDirname(backup_dirs, backup_dirs[i].path); + } + + row.push_back(backup_dirs[i].name); + + if (has_group) + { + row.push_back(convert(backup_dirs[i].group)); + } + + if (has_virtual_client) + { + if (backup_dirs[i].virtual_client.empty()) + { + row.push_back("-"); + } + else + { + row.push_back(backup_dirs[i].virtual_client); + } + } + + row.push_back(backup_dirs[i].flags); + + if (has_server_default) + { + if (backup_dirs[i].server_default) + { + row.push_back("Yes"); + } + else + { + row.push_back("No"); + } + } + + tab.push_back(row); + } + + display_table(tab); + + return 0; +} + +int action_remove_backupdir(std::vector args) +{ + TCLAP::CmdLine cmd("Remove directory from backup set", ' ', cmdline_version); + + PwClientCmd pw_client_cmd(cmd, true); + + TCLAP::ValueArg name_arg("n", "name", + "Backup directory name", + false, "", "name"); + + TCLAP::ValueArg path_arg("d", "path", + "Backup path", + true, "", "path"); + + cmd.xorAdd(name_arg, path_arg); + + cmd.parse(args); + + if (!pw_client_cmd.set()) + { + return 3; + } + + std::vector backup_dirs = Connector::getSharedPaths(true); + + if (Connector::hasError()) + { + std::cerr << "Error retrieving current backup directories from backend" << std::endl; + return 1; + } + + bool del_ok = false; + bool del_server_default = false; + + for (size_t i = 0; i < backup_dirs.size();) + { + if (!name_arg.getValue().empty() + && backup_dirs[i].name == name_arg.getValue()) + { + if (backup_dirs[i].server_default) + { + del_server_default = true; + ++i; + } + else + { + backup_dirs.erase(backup_dirs.begin() + i); + del_ok = true; + } + } + else if (!path_arg.getValue().empty() + && backup_dirs[i].path == path_arg.getValue()) + { + if (backup_dirs[i].server_default) + { + del_server_default = true; + ++i; + } + else + { + backup_dirs.erase(backup_dirs.begin() + i); + del_ok = true; + } + } + else + { + ++i; + } + } + + if (!del_ok) + { + if (del_server_default) + { + std::cerr << "Backup directory was configured on the server. Please remove it there" << std::endl; + } + else + { + std::cerr << "Backup directory to remove not found" << std::endl; + } + return 1; + } + + if (!Connector::saveSharedPaths(backup_dirs)) + { + std::cerr << "Error removing backup directory via backend" << std::endl; + return 2; + } + else + { + return 0; + } +} + +int action_wait_for_backend(std::vector args) +{ + TCLAP::CmdLine cmd("Wait for backend to become available", ' ', cmdline_version); + + PwClientCmd pw_client_cmd(cmd, false); + + TCLAP::ValueArg time_arg("t", "time", + "Max time in seconds to wait", + false, 60, "seconds", cmd); + + cmd.parse(args); + + pw_client_cmd.wait(time_arg.getValue() * 1000); + + if (!pw_client_cmd.set()) + { + return 3; + } + + int64 starttime = getTimeMS(); + do + { + int64 thistime = getTimeMS(); + std::string d = Connector::getStatusRawNoWait(); + if (!Connector::hasError() + && !d.empty()) + { + return 0; + } + if (getTimeMS() - thistime < 30) + { + wait(100); + } + } while (getTimeMS() - starttime < time_arg.getValue() * 1000); + + std::cerr << "Could not connect to backend in specified time" << std::endl; + return 1; +} + +int main(int argc, char *argv[]) +{ + if(argc==0) + { + std::cerr << "Not enough arguments (zero arguments) -- no program name" << std::endl; + return 1; + } + +#ifdef _WIN32 + HMODULE hModule = GetModuleHandleW(NULL); + if (hModule != INVALID_HANDLE_VALUE) + { + WCHAR path[MAX_PATH]; + if (GetModuleFileNameW(hModule, path, MAX_PATH) != 0) + { + SetCurrentDirectoryW(path); + } + } +#endif + + std::vector actions; + std::vector action_funs; + actions.push_back("start"); + action_funs.push_back(action_start); + actions.push_back("status"); + action_funs.push_back(action_status); + actions.push_back("browse"); + action_funs.push_back(action_browse); + actions.push_back("restore-start"); + action_funs.push_back(action_start_restore); + actions.push_back("set-settings"); + action_funs.push_back(action_set_settings); + actions.push_back("reset-keep"); + action_funs.push_back(action_reset_keep); + actions.push_back("add-backupdir"); + action_funs.push_back(action_add_backupdir); + actions.push_back("list-backupdirs"); + action_funs.push_back(action_list_backupdirs); + actions.push_back("remove-backupdir"); + action_funs.push_back(action_remove_backupdir); + actions.push_back("wait-for-backend"); + action_funs.push_back(action_wait_for_backend); + + bool has_help=false; + bool has_version=false; + size_t action_idx=std::string::npos; + std::vector args; + args.push_back(argv[0]); + for(int i=1;i osx-pkg/Library/Application\ Support/UrBackup\ Client/etc/urbackup/snapshot.cfg +echo "remove_filesystem_snapshot=/Library/Application\ Support/UrBackup\ Client/etc/urbackup/apfs_remove_snapshot" >> osx-pkg/Library/Application\ Support/UrBackup\ Client/etc/urbackup/snapshot.cfg + +cp linux_snapshot/apfs_create_snapshot osx-pkg/Library/Application\ Support/UrBackup\ Client/etc/urbackup/apfs_create_snapshot +cp linux_snapshot/apfs_remove_snapshot osx-pkg/Library/Application\ Support/UrBackup\ Client/etc/urbackup/apfs_remove_snapshot + +if !($development); then + strip "osx-pkg2/Applications/UrBackup Client.app/Contents/MacOS/urbackupclientgui" + strip "osx-pkg2/Applications/UrBackup Client.app/Contents/MacOS/sbin/urbackupclientbackend" +fi mkdir -p "$PWD/osx-pkg2/Applications/UrBackup Client.app/Contents/MacOS/sbin" UNINSTALLER="$PWD/osx-pkg2/Applications/UrBackup Client.app/Contents/MacOS/sbin/urbackup_uninstall" @@ -75,53 +138,34 @@ echo "OK=true" >> "$UNINSTALLER" chmod +x "$UNINSTALLER" +GIT_REV="$(git rev-parse --short HEAD)" +if [ $? -ne 0 ]; then + GIT_REV="N/A" +fi + if !($development); then - VERSION_SHORT_NUM="$version_num_short$" + VERSION_SHORT_NUM="$version_num_short$ ($GIT_REV)" VERSION_SHORT="$version_short$" else - VERSION_SHORT_NUM="0.1" + VERSION_SHORT_NUM="0.1 ($GIT_REV)" VERSION_SHORT="0.1" fi -function notarization_info { - echo "$UPLOAD_INFO_PLIST" > tmp.plist - xcrun altool --notarization-info `/usr/libexec/PlistBuddy -c "Print :notarization-upload:RequestUUID" tmp.plist` -u "$AC_USERNAME" -p "@env:AC_PASSWORD" --output-format xml -} - -function wait_for_notarization { - echo "Waiting for notarization to finish..." - sleep 30 - while true; do - REQUEST_INFO_PLIST=$(notarization_info || true) - echo "$REQUEST_INFO_PLIST" > tmp.plist - if [ "x$(/usr/libexec/PlistBuddy -c 'Print :product-errors:0:code' tmp.plist)" = x1519 ]; then - sleep 30 - continue - fi - if [ "x$(/usr/libexec/PlistBuddy -c 'Print :notarization-info:Status' tmp.plist)" != "xin progress" ]; then - echo "Notarization finished" - break - fi - sleep 60 - done - -} - -function notarize_int { - xcrun altool --notarize-app --primary-bundle-id "org.urbackup.client.frontend" -u "$AC_USERNAME" -p "@env:AC_PASSWORD" -t osx -f "$1" --output-format xml -} +if ($development); then + gsed -i 's/\$version_num_short\$/0.1/g' "osx-pkg2/Applications/UrBackup Client.app/Contents/Info.plist" + gsed -i 's/\$version_maj\$/0/g' "osx-pkg2/Applications/UrBackup Client.app/Contents/Info.plist" + gsed -i 's/\$version_min\$/1/g' "osx-pkg2/Applications/UrBackup Client.app/Contents/Info.plist" +fi +gsed -i 's/\$git_rev\$/'"$GIT_REV"'/g' "osx-pkg2/Applications/UrBackup Client.app/Contents/Info.plist" function notarize { echo "Sending $1 to notarization..." - UPLOAD_INFO_PLIST=$(notarize_int $1) - echo $UPLOAD_INFO_PLIST - wait_for_notarization + xcrun notarytool submit "$1" --keychain-profile "notary-profile" --wait } if !($development); then echo "Signing code..." - security unlock-keychain -p foobar /Users/martin/Library/Keychains/dev.keychain - codesign --deep --keychain dev.keychain --sign 3Y4WACCWC5 --timestamp --options runtime osx-pkg2/Applications/UrBackup\ Client.app + codesign --deep --sign 3Y4WACCWC5 --timestamp --options runtime osx-pkg2/Applications/UrBackup\ Client.app ditto -c -k --keepParent "osx-pkg2/Applications" "urbackup-client.zip" notarize "urbackup-client.zip" xcrun stapler staple "osx-pkg2/Applications/UrBackup Client.app" @@ -129,12 +173,12 @@ fi rm -R pkg1 || true mkdir pkg1 || true -pkgbuild --root osx-pkg --identifier org.urbackup.client.service --version $VERSION_SHORT_NUM --ownership recommended pkg1/output.pkg -pkgbuild --root "osx-pkg2/Applications/UrBackup Client.app" --identifier "org.urbackup.client" --version $VERSION_SHORT_NUM --scripts osx_installer/scripts2 --ownership recommended pkg1/output2.pkg --install-location "/Applications/UrBackup Client.app" -productbuild --distribution osx_installer/distribution.xml --resources osx_installer/resources --package-path pkg1 --version $VERSION_SHORT_NUM final.pkg +pkgbuild --root osx-pkg --identifier org.urbackup.client.service --version "$VERSION_SHORT_NUM" --ownership recommended pkg1/output.pkg +pkgbuild --root "osx-pkg2/Applications/UrBackup Client.app" --identifier "org.urbackup.client" --version "$VERSION_SHORT_NUM" --scripts osx_installer/scripts2 --ownership recommended pkg1/output2.pkg --install-location "/Applications/UrBackup Client.app" +productbuild --distribution osx_installer/distribution.xml --resources osx_installer/resources --package-path pkg1 --version "$VERSION_SHORT_NUM" final.pkg if !($development); then - productsign --keychain /Users/martin/Library/Keychains/dev.keychain --sign 3Y4WACCWC5 final.pkg final-signed.pkg + productsign --sign 3Y4WACCWC5 final.pkg final-signed.pkg notarize final-signed.pkg xcrun stapler staple final-signed.pkg diff --git a/cryptoplugin/CryptoFactory.cpp b/cryptoplugin/CryptoFactory.cpp index c690be96a..ffb5a6d63 100644 --- a/cryptoplugin/CryptoFactory.cpp +++ b/cryptoplugin/CryptoFactory.cpp @@ -19,7 +19,9 @@ #include "../vld.h" #include "CryptoFactory.h" #include "../Interface/Server.h" +#include "../Interface/File.h" #include "../Interface/ThreadPool.h" +#include "../stringtools.h" #include "AESEncryption.h" #include "AESDecryption.h" @@ -30,6 +32,7 @@ #include "cryptopp_inc.h" #include "ECDHKeyExchange.h" +#include using namespace CryptoPPCompat; @@ -422,4 +425,55 @@ std::string CryptoFactory::sha256Binary(const std::string& data) byte sha256_digest[CryptoPP::SHA256::DIGESTSIZE]; CryptoPP::SHA256().CalculateDigest(sha256_digest, reinterpret_cast(data.data()), data.size()); return std::string(reinterpret_cast(sha256_digest), sizeof(sha256_digest)); -} \ No newline at end of file +} + +bool CryptoFactory::convertOpenSslSig(const std::string& pubkeyFn, const std::string& sigFn, const std::string& outFn) +{ + std::string derSignature, p1363Signature; + derSignature = getFile(sigFn); + if (derSignature.empty()) + { + Server->Log("Error reading sigFn at " + sigFn, LL_ERROR); + return false; + } + + CryptoPP::ECDSA::PublicKey PublicKey; + + try + { + PublicKey.Load(CryptoPP::FileSource(pubkeyFn.c_str(), true).Ref()); + + CryptoPP::ECDSA::Verifier verifier(PublicKey); + + // Make room for the P1363 encoding + p1363Signature.resize(verifier.SignatureLength()); + + size_t encodedSize = CryptoPP::DSAConvertSignatureFormat( + (byte*)(&p1363Signature[0]), p1363Signature.size(), CryptoPP::DSA_P1363, + (const byte*)(derSignature.data()), derSignature.size(), CryptoPP::DSA_DER); + + assert(encodedSize <= p1363Signature.size()); + p1363Signature.resize(encodedSize); + + std::auto_ptr f(Server->openFile(outFn, MODE_WRITE)); + if (f.get() == NULL) + { + Server->Log("Error opening out file for signature at " + outFn, LL_ERROR); + return false; + } + + if (f->Write(p1363Signature) != p1363Signature.size()) + { + Server->Log("Error writing signature to " + outFn, LL_ERROR); + return false; + } + + return true; + } + catch (const CryptoPP::Exception& e) + { + Server->Log("Exception occured in CryptoFactory::convertOpenSslSig: " + e.GetWhat(), LL_ERROR); + } + + return false; +} diff --git a/cryptoplugin/CryptoFactory.h b/cryptoplugin/CryptoFactory.h index a0fc395ec..2b00af45d 100644 --- a/cryptoplugin/CryptoFactory.h +++ b/cryptoplugin/CryptoFactory.h @@ -29,5 +29,5 @@ class CryptoFactory : public ICryptoFactory virtual IECDHKeyExchange* createECDHKeyExchange(); virtual std::string sha1Binary(const std::string& data); virtual std::string sha256Binary(const std::string& data); - + virtual bool convertOpenSslSig(const std::string& pubkeyFn, const std::string& sigFn, const std::string& outFn) override; }; \ No newline at end of file diff --git a/cryptoplugin/ICryptoFactory.h b/cryptoplugin/ICryptoFactory.h index 322007ed3..8910f6434 100644 --- a/cryptoplugin/ICryptoFactory.h +++ b/cryptoplugin/ICryptoFactory.h @@ -44,4 +44,6 @@ class ICryptoFactory: public IPlugin virtual std::string sha1Binary(const std::string& data) = 0; virtual std::string sha256Binary(const std::string& data) = 0; + + virtual bool convertOpenSslSig(const std::string& pubkeyFn, const std::string& sigFn, const std::string& outFn) = 0; }; diff --git a/cryptoplugin/cryptoplugin.vcxproj b/cryptoplugin/cryptoplugin.vcxproj index 71ed9bbcb..bab5bf4d7 100644 --- a/cryptoplugin/cryptoplugin.vcxproj +++ b/cryptoplugin/cryptoplugin.vcxproj @@ -1,6 +1,10 @@  + + Debug + ARM64 + Debug Win32 @@ -9,6 +13,10 @@ Debug x64 + + Release + ARM64 + Release Win32 @@ -29,23 +37,34 @@ DynamicLibrary Unicode true - v142 + v143 DynamicLibrary Unicode true - v142 + v143 + + + DynamicLibrary + Unicode + true + v143 DynamicLibrary Unicode - v142 + v143 DynamicLibrary Unicode - v142 + v143 + + + DynamicLibrary + Unicode + v143 @@ -56,39 +75,77 @@ + + + + + + <_ProjectFileVersion>10.0.30319.1 $(SolutionDir)$(Configuration)\ $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ $(Configuration)\ $(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ true true + true $(SolutionDir)$(Configuration)\ $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ $(Configuration)\ $(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ false false + false AllRules.ruleset AllRules.ruleset + AllRules.ruleset + + AllRules.ruleset AllRules.ruleset + AllRules.ruleset + + + + + true + + + x64-windows-static-md + Debug + + + arm64-windows-static-md + Debug + + + x64-windows-static-md + + + x64-windows-static-md + + + x86-windows-static-md @@ -115,7 +172,26 @@ Disabled - $(CryptoppIncludeDir);$(SolutionDir)/deps/include/cryptopp;%(AdditionalIncludeDirectories) + %(AdditionalIncludeDirectories) + WIN32;_DEBUG;_WINDOWS;_USRDLL;CRYPTOPLUGIN_EXPORTS;%(PreprocessorDefinitions) + EnableFastChecks + MultiThreadedDebugDLL + + + Level3 + ProgramDatabase + + + %(AdditionalDependencies) + $(CryptoppLibDir);$(SolutionDir)/deps/libs;%(AdditionalLibraryDirectories) + true + Windows + + + + + Disabled + %(AdditionalIncludeDirectories) WIN32;_DEBUG;_WINDOWS;_USRDLL;CRYPTOPLUGIN_EXPORTS;%(PreprocessorDefinitions) EnableFastChecks MultiThreadedDebugDLL @@ -125,7 +201,7 @@ ProgramDatabase - cryptlibd_x86_64.lib;%(AdditionalDependencies) + %(AdditionalDependencies) $(CryptoppLibDir);$(SolutionDir)/deps/libs;%(AdditionalLibraryDirectories) true Windows @@ -151,7 +227,7 @@ true MachineX86 $(CryptoppLibDir);$(SolutionDir)deps\libs;%(AdditionalLibraryDirectories) - cryptlib_x86.lib;%(AdditionalDependencies) + %(AdditionalDependencies) @@ -172,8 +248,30 @@ Windows true true - $(CryptoppLibDir);$(SolutionDir)deps\libs;%(AdditionalLibraryDirectories) - cryptlib_x86_64.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + %(AdditionalDependencies) + + + + + MaxSpeed + true + WIN32;NDEBUG;_WINDOWS;_USRDLL;CRYPTOPLUGIN_EXPORTS;%(PreprocessorDefinitions) + MultiThreadedDLL + true + + + Level3 + ProgramDatabase + $(CryptoppIncludeDir);$(SolutionDir)/deps/include/cryptopp; + + + true + Windows + true + true + %(AdditionalLibraryDirectories) + %(AdditionalDependencies) diff --git a/cryptoplugin/cryptopp_inc.h b/cryptoplugin/cryptopp_inc.h index 4bc40c978..638f7ee9f 100644 --- a/cryptoplugin/cryptopp_inc.h +++ b/cryptoplugin/cryptopp_inc.h @@ -1,21 +1,8 @@ #ifdef _WIN32 -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#define CRYPTOPP_INCLUDE_PREFIX cryptopp #else #include "../config.h" +#endif #define CRYPTOPP_INCLUDE_AES #define CRYPTOPP_INCLUDE_SHA #define CRYPTOPP_INCLUDE_MODES @@ -49,7 +36,6 @@ #if (CRYPTOPP_VERSION >= 564) #include CRYPTOPP_INCLUDE_CRC #endif -#endif namespace CryptoPPCompat { diff --git a/cryptoplugin/dllmain.cpp b/cryptoplugin/dllmain.cpp index 4e427736d..d343fcb54 100644 --- a/cryptoplugin/dllmain.cpp +++ b/cryptoplugin/dllmain.cpp @@ -23,6 +23,7 @@ #define DLLEXPORT extern "C" #include #endif +#include #ifndef STATIC_PLUGIN @@ -69,10 +70,12 @@ DLLEXPORT void LoadActions(IServer* pServer) if(b) { Server->Log("Keys generated successfully", LL_INFO); + exit(0); } else { Server->Log("Generating keys failed", LL_ERROR); + exit(1); } } else if(crypto_action=="sign_file") @@ -85,10 +88,12 @@ DLLEXPORT void LoadActions(IServer* pServer) if(!b) { Server->Log("Signing file failed", LL_INFO); + exit(1); } else { Server->Log("Signed file successfully", LL_INFO); + exit(0); } } else if(crypto_action=="sign_file_dsa") @@ -101,10 +106,12 @@ DLLEXPORT void LoadActions(IServer* pServer) if(!b) { Server->Log("Signing file failed", LL_INFO); + exit(1); } else { Server->Log("Signed file successfully (DSA)", LL_INFO); + exit(0); } } else if(crypto_action=="verify_file") @@ -117,15 +124,36 @@ DLLEXPORT void LoadActions(IServer* pServer) if(!b) { Server->Log("Verifying file failed", LL_INFO); + exit(1); } else { - Server->Log("Verfifed file successfully", LL_INFO); + Server->Log("Verified file successfully", LL_INFO); + exit(0); + } + } + else if (crypto_action == "convert_sig") + { + std::string pubkey_fn = Server->getServerParameter("pubkey_fn"); + std::string sig_fn = Server->getServerParameter("sig_fn"); + std::string sig_out_fn = Server->getServerParameter("sig_out_fn"); + CryptoFactory fak; + const bool b = fak.convertOpenSslSig(pubkey_fn, sig_fn, sig_out_fn); + if (!b) + { + Server->Log("Converting signature failed", LL_INFO); + exit(1); + } + else + { + Server->Log("Converted signature successfully", LL_INFO); + exit(0); } } else { Server->Log("Unknown crypto_action"); + exit(1); } } diff --git a/defaults_server b/defaults_server index 07bc9b1ec..495ca4e52 100755 --- a/defaults_server +++ b/defaults_server @@ -10,12 +10,20 @@ FASTCGI_PORT=55413 #Enable internal HTTP server +# Required for serving web interface without FastCGI +# and for websocket connections from client HTTP_SERVER="true" #Port for the web interface #(if internal HTTP server is enabled) HTTP_PORT=55414 +#Bind HTTP server to localhost only +HTTP_LOCALHOST_ONLY=false + +#Bind Internet port to localhost only +INTERNET_LOCALHOST_ONLY=false + #log file name LOGFILE="/var/log/urbackup.log" @@ -35,5 +43,8 @@ SQLITE_TMPDIR="" #Comma separated -- e.g. "eth0,eth1" BROADCAST_INTERFACES="" +# Enable better error messages if a user cannot be found during login +ALLOW_USER_ENUMERATION="true" + #User the urbackupsrv process runs as USER="urbackup" diff --git a/dmsnaptool/main.cpp b/dmsnaptool/main.cpp new file mode 100644 index 000000000..f94a59cdd --- /dev/null +++ b/dmsnaptool/main.cpp @@ -0,0 +1,431 @@ +/************************************************************************* +* UrBackup - Client/Server backup system +* Copyright (C) 2011-2017 Martin Raiber +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +**************************************************************************/ + +#include +#include "../tclap/CmdLine.h" +#include +#include "../stringtools.h" +#include +#include +#include + +std::string cmdline_version = "1.0"; + +int64_t getTimeMS() +{ + + timespec tp; + if (clock_gettime(CLOCK_MONOTONIC, &tp) != 0) + { + timeval tv; + gettimeofday(&tv, NULL); + static long start_t = tv.tv_sec; + tv.tv_sec -= start_t; + return tv.tv_sec * 1000 + tv.tv_usec / 1000; + } + return static_cast(tp.tv_sec) * 1000 + tp.tv_nsec / 1000000; +} + +std::string strip_mapper(std::string dev) +{ + if (dev.find("/dev/mapper/") == 0) + { + dev.erase(0, 12); + } + return dev; +} + +bool do_task_simple(int task, const std::string& dev) +{ + struct dm_task* dmt; + if (!(dmt = dm_task_create(task))) + return false; + + if (!dm_task_set_name(dmt, strip_mapper(dev).c_str())) + return false; + + if (!dm_task_run(dmt)) + return false; + + dm_task_destroy(dmt); + return true; +} + +bool do_create_dm_dev(uint64_t start, uint64_t size, const std::string& ttype, const std::string& params, bool readonly, const std::string& dev, uint32_t cookie) +{ + struct dm_task* dmt; + if (!(dmt = dm_task_create(DM_DEVICE_CREATE))) + return false; + + if (!dm_task_set_name(dmt, strip_mapper(dev).c_str())) + return false; + + if (!dm_task_set_cookie(dmt, &cookie, DM_UDEV_DISABLE_LIBRARY_FALLBACK)) + return false; + + if (readonly && !dm_task_set_ro(dmt)) + return false; + + if (!dm_task_add_target(dmt, start, size, ttype.c_str(), params.c_str())) + return false; + + if (!dm_task_set_add_node(dmt, DM_ADD_NODE_ON_RESUME)) + return false; + + if (!dm_task_run(dmt)) + { + std::cerr << "Error creating device" << std::endl; + return false; + } + + dm_task_destroy(dmt); + + return true; +} + +bool do_reload_dm_dev(uint64_t start, uint64_t size, const std::string& ttype, const std::string& params, bool readonly, const std::string& dev) +{ + struct dm_task* dmt; + if (!(dmt = dm_task_create(DM_DEVICE_RELOAD))) + return false; + + if (!dm_task_set_name(dmt, strip_mapper(dev).c_str())) + return false; + + if (readonly && !dm_task_set_ro(dmt)) + return false; + + if (!dm_task_add_target(dmt, start, size, ttype.c_str(), params.c_str())) + return false; + + if (!dm_task_run(dmt)) + { + std::cerr << "Error creating device" << std::endl; + return false; + } + + dm_task_destroy(dmt); + + return true; +} + +bool do_message(const std::string& dev, const std::string& msg) +{ + struct dm_task* dmt; + if (!(dmt = dm_task_create(DM_DEVICE_RELOAD))) + return false; + + if (!dm_task_set_name(dmt, strip_mapper(dev).c_str())) + return false; + + if (!dm_task_set_sector(dmt, 0)) + return false; + + if (!dm_task_set_message(dmt, msg.c_str())) + return false; + + if (!dm_task_run(dmt)) + { + std::cerr << "Error sending message" << std::endl; + return false; + } + dm_task_destroy(dmt); + + return true; +} + +bool has_snapshot_origin(const std::string& dev, bool& ret, uint64_t& start, uint64_t& length, std::string& params) +{ + struct dm_task* dmt; + if (!(dmt = dm_task_create(DM_DEVICE_TABLE))) + return false; + + if (!dm_task_set_name(dmt, strip_mapper(dev).c_str())) + return false; + + if (!dm_task_run(dmt)) + { + std::cerr << "Error getting device table" << std::endl; + return false; + } + + struct dm_info info; + if (!dm_task_get_info(dmt, &info) || !info.exists) + { + std::cerr << "Error getting device info" << std::endl; + return false; + } + + ret = false; + + void* next = NULL; + do { + char* target_type, * ptr_params; + next = dm_get_next_target(dmt, next, &start, &length, + &target_type, &ptr_params); + + ret = std::string(target_type) == "snapshot-origin"; + if (ptr_params != NULL) + params = ptr_params; + + } while (next); + + dm_task_destroy(dmt); + + return true; +} + +bool get_era(const std::string& dev, int64_t& era) +{ + struct dm_task* dmt; + if (!(dmt = dm_task_create(DM_DEVICE_STATUS))) + return false; + + if (!dm_task_set_name(dmt, strip_mapper(dev).c_str())) + return false; + + if (!dm_task_run(dmt)) + { + std::cerr << "Error getting device table" << std::endl; + return false; + } + + struct dm_info info; + if (!dm_task_get_info(dmt, &info) || !info.exists) + { + std::cerr << "Error getting device info for getting era" << std::endl; + return false; + } + + bool ret = false; + void* next = NULL; + do { + uint64_t start, length; + char* target_type, * ptr_params; + next = dm_get_next_target(dmt, next, &start, &length, + &target_type, &ptr_params); + + if (std::string(target_type) == "era") + { + std::vector toks; + Tokenize(ptr_params, toks, " "); + + if (toks.size() < 4) + { + ret = false; + std::cerr << "Not enough status" << std::endl; + } + else + { + era = watoi64(toks[2]); + if (convert(static_cast(era)) != toks[2]) + { + std::cerr << "Unable to convert era to number" << std::endl; + } + } + } + + } while (next); + + dm_task_destroy(dmt); + + return ret; +} + +struct DMResumeDev +{ + DMResumeDev(std::string dev, uint32_t cookie) + : dev(dev), cookie(cookie) + {} + + ~DMResumeDev() + { + if (!do_task_simple(DM_DEVICE_RESUME, dev)) + { + std::cerr << "Error resuming dev " << dev << std::endl; + } + + if (!dm_udev_wait(cookie)) + { + std::cerr << "Error waiting for udev cookie release" << std::endl; + } + } + + std::string dev; + uint32_t cookie; +}; + +bool do_create_dm_snapshot(const std::string& dev, const std::string& dev_clone, const std::string& dev_snap, const std::string& cow_dev, const std::string& dev_origin, const std::string& era_access_dev, uint64_t dev_size) +{ + int rc = mlockall(MCL_CURRENT | MCL_FUTURE); + if (rc != 0) + { + perror("mlockall"); + return false; + } + + uint32_t cookie = 0; + + if (!dm_udev_create_cookie(&cookie)) + { + std::cerr << "Error creating udev cookie" << std::endl; + return false; + } + + if (!do_task_simple(DM_DEVICE_SUSPEND, dev)) + { + if (!dm_udev_wait(cookie)) + { + std::cerr << "Error waiting for udev cookie release (1)" << std::endl; + } + return false; + } + + int64_t init_era; + if (get_era(dev_clone, init_era)) + { + std::cout << "Checkpointing device for CBT..." << std::endl; + + if (!do_message(dev_clone, "checkpoint")) + { + if (!dm_udev_wait(cookie)) + { + std::cerr << "Error waiting for udev cookie release (2)" << std::endl; + } + return false; + } + + int64_t starttime = getTimeMS(); + + bool era_update = false; + while (getTimeMS() - starttime < 60000) + { + int64_t curr_era; + if (get_era(dev_clone, curr_era) + && curr_era > init_era) + { + era_update = true; + break; + } + + usleep(100000); + } + + if (era_update) + { + std::cout << "CBT=type=era" << std::endl; + std::cout << "CBT_FILE=" << era_access_dev << std::endl; + } + } + + DMResumeDev resume_dev(dev, cookie); + + + std::cout << "Creating snapshot device..." << std::endl; + + if (!do_create_dm_dev(0, dev_size, "snapshot", dev_clone + " " + cow_dev + " N 32", true, dev_snap, cookie)) + return false; + + struct stat statbuf; + if (stat(dev_origin.c_str(), &statbuf)!=0 && errno==ENOENT) + { + std::cout << "Creating snapshot origin device..." << std::endl; + if (!do_create_dm_dev(0, dev_size, "snapshot-origin", dev_clone, false, dev_origin, cookie)) + return false; + } + + bool b_origin; + uint64_t start, length; + std::string origin_params; + if (!has_snapshot_origin(dev_origin, b_origin, start, length, origin_params)) + return false; + + if(start!=0 && length!=dev_size) + { + std::cerr << "Start (" << start << ") or length (" << length << ") of origin dev wrong. Expected " + << "0, " << dev_size << std::endl; + return false; + } + + if(!b_origin) + { + std::cerr << "Origin dev " << dev_origin << " is not snapshot-origin" << std::endl; + return false; + } + + bool b_snapshot_origin; + uint64_t tmp_start, tmp_length; + std::string tmp_params; + if (!has_snapshot_origin(dev, b_snapshot_origin, tmp_start, tmp_length, tmp_params)) + return false; + + if (!b_snapshot_origin) + { + if (!do_reload_dm_dev(0, dev_size, "snapshot-origin", origin_params, false, dev)) + return false; + } + + return true; +} + +int main(int argc, char* argv[]) +{ + if(argc==0) + { + std::cout << "Not enough arguments (zero arguments) -- no program name" << std::endl; + return 1; + } + try + { + TCLAP::CmdLine cmd("Create snapshot of dm-snapshot root volume", ' ', cmdline_version); + + //bool do_create_dm_snapshot(const std::string& dev, const std::string& dev_clone, const std::string& dev_snap, const std::string& cow_dev, const std::string& dev_origin, const std::string& era_access_dev, uint64_t dev_size) + + TCLAP::ValueArg dev_arg("","dev","Device to snapshot",true,"","string"); + TCLAP::ValueArg clone_dev_arg("","clone-dev","Clone of device to snapshot",true,"","string"); + TCLAP::ValueArg snap_dev_arg("","snap-dev","Name of snapshot device",true,"","string"); + TCLAP::ValueArg cow_dev_arg("","cow-dev","COW snapshot storage device",true,"","string"); + TCLAP::ValueArg origin_dev_arg("","origin-dev","Snapshot origin device",true,"","string"); + TCLAP::ValueArg era_access_dev_arg("","era-access-dev","Device for ERA access",true,"","string"); + TCLAP::ValueArg dev_size_arg("","dev-size","Device for ERA access",true,0,"int"); + + cmd.add(dev_arg); + cmd.add(clone_dev_arg); + cmd.add(snap_dev_arg); + cmd.add(cow_dev_arg); + cmd.add(origin_dev_arg); + cmd.add(era_access_dev_arg); + cmd.add(dev_size_arg); + + cmd.parse(argc, argv); + + bool r = do_create_dm_snapshot(dev_arg.getValue(), clone_dev_arg.getValue(), snap_dev_arg.getValue(), cow_dev_arg.getValue(), origin_dev_arg.getValue(), era_access_dev_arg.getValue(), dev_size_arg.getValue()); + + if(!r) + { + std::cerr << "Creating dm-snapshot failed" << std::endl; + exit(1); + } + + exit(0); + } + catch (TCLAP::ArgException &e) + { + std::cerr << "error: " << e.error() << " for arg " << e.argId() << std::endl; + } +} diff --git a/external/imdisk/imdisk.h b/external/imdisk/imdisk.h new file mode 100644 index 000000000..8a1719d0b --- /dev/null +++ b/external/imdisk/imdisk.h @@ -0,0 +1,1361 @@ +/* +ImDisk Virtual Disk Driver for Windows NT/2000/XP. + +Copyright (C) 2005-2015 Olof Lagerkvist. + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. +*/ + +#ifndef _INC_IMDISK_ +#define _INC_IMDISK_ + +#ifndef __T +#if defined(_NTDDK_) || defined(UNICODE) || defined(_UNICODE) +#define __T(x) L ## x +#else +#define __T(x) x +#endif +#endif + +#ifndef _T +#define _T(x) __T(x) +#endif + +#include "imdiskver.h" +#define IMDISK_VERSION ((IMDISK_MAJOR_VERSION << 8) | (IMDISK_MINOR_VERSION << 4) | (IMDISK_MINOR_LOW_VERSION)) +#define IMDISK_DRIVER_VERSION 0x0103 + +#ifndef ZERO_STRUCT +#define ZERO_STRUCT { 0 } +#endif + +/// +/// Base names for device objects created in \Device +/// +#define IMDISK_DEVICE_DIR_NAME _T("\\Device") +#define IMDISK_DEVICE_BASE_NAME IMDISK_DEVICE_DIR_NAME _T("\\ImDisk") +#define IMDISK_CTL_DEVICE_NAME IMDISK_DEVICE_BASE_NAME _T("Ctl") + +/// +/// Symlinks created in \DosDevices to device objects +/// +#define IMDISK_SYMLNK_NATIVE_DIR_NAME _T("\\DosDevices") +#define IMDISK_SYMLNK_WIN32_DIR_NAME _T("\\\\?") +#define IMDISK_SYMLNK_NATIVE_BASE_NAME IMDISK_SYMLNK_NATIVE_DIR_NAME _T("\\ImDisk") +#define IMDISK_SYMLNK_WIN32_BASE_NAME IMDISK_SYMLNK_WIN32_DIR_NAME _T("\\ImDisk") +#define IMDISK_CTL_SYMLINK_NAME IMDISK_SYMLNK_NATIVE_BASE_NAME _T("Ctl") +#define IMDISK_CTL_DOSDEV_NAME IMDISK_SYMLNK_WIN32_BASE_NAME _T("Ctl") + +/// +/// The driver name and image path +/// +#define IMDISK_DRIVER_NAME _T("ImDisk") +#define IMDISK_DRIVER_PATH _T("system32\\drivers\\imdisk.sys") + +#ifndef AWEALLOC_DRIVER_NAME +#define AWEALLOC_DRIVER_NAME _T("AWEAlloc") +#endif +#ifndef AWEALLOC_DEVICE_NAME +#define AWEALLOC_DEVICE_NAME _T("\\Device\\") AWEALLOC_DRIVER_NAME +#endif + +/// +/// Global refresh event name +/// +#define IMDISK_REFRESH_EVENT_NAME _T("ImDiskRefresh") + +/// +/// Registry settings. It is possible to specify devices to be mounted +/// automatically when the driver loads. +/// +#define IMDISK_CFG_PARAMETER_KEY _T("\\Parameters") +#define IMDISK_CFG_MAX_DEVICES_VALUE _T("MaxDevices") +#define IMDISK_CFG_LOAD_DEVICES_VALUE _T("LoadDevices") +#define IMDISK_CFG_DISALLOWED_DRIVE_LETTERS_VALUE _T("DisallowedDriveLetters") +#define IMDISK_CFG_IMAGE_FILE_PREFIX _T("FileName") +#define IMDISK_CFG_SIZE_PREFIX _T("Size") +#define IMDISK_CFG_FLAGS_PREFIX _T("Flags") +#define IMDISK_CFG_DRIVE_LETTER_PREFIX _T("DriveLetter") +#define IMDISK_CFG_OFFSET_PREFIX _T("ImageOffset") + +#define KEY_NAME_HKEY_MOUNTPOINTS \ + _T("Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\MountPoints") +#define KEY_NAME_HKEY_MOUNTPOINTS2 \ + _T("Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\MountPoints2") + +#define IMDISK_WINVER_MAJOR() (GetVersion() & 0xFF) +#define IMDISK_WINVER_MINOR() ((GetVersion() & 0xFF00) >> 8) + +#define IMDISK_WINVER() ((IMDISK_WINVER_MAJOR() << 8) | \ + IMDISK_WINVER_MINOR()) + +#if defined(NT4_COMPATIBLE) && !defined(_WIN64) +#define IMDISK_GTE_WIN2K() (IMDISK_WINVER_MAJOR() >= 0x05) +#else +#define IMDISK_GTE_WIN2K() TRUE +#endif + +#ifdef _WIN64 +#define IMDISK_GTE_WINXP() TRUE +#else +#define IMDISK_GTE_WINXP() (IMDISK_WINVER() >= 0x0501) +#endif + +#define IMDISK_GTE_SRV2003() (IMDISK_WINVER() >= 0x0502) + +#define IMDISK_GTE_VISTA() (IMDISK_WINVER_MAJOR() >= 0x06) + +#ifndef IMDISK_API +#ifdef IMDISK_CPL_EXPORTS +#define IMDISK_API __declspec(dllexport) +#else +#define IMDISK_API __declspec(dllimport) +#endif +#endif + +/// +/// Base value for the IOCTL's. +/// +#define FILE_DEVICE_IMDISK 0x8372 + +#define IOCTL_IMDISK_QUERY_VERSION ((ULONG) CTL_CODE(FILE_DEVICE_IMDISK, 0x800, METHOD_BUFFERED, 0)) +#define IOCTL_IMDISK_CREATE_DEVICE ((ULONG) CTL_CODE(FILE_DEVICE_IMDISK, 0x801, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS)) +#define IOCTL_IMDISK_QUERY_DEVICE ((ULONG) CTL_CODE(FILE_DEVICE_IMDISK, 0x802, METHOD_BUFFERED, 0)) +#define IOCTL_IMDISK_QUERY_DRIVER ((ULONG) CTL_CODE(FILE_DEVICE_IMDISK, 0x803, METHOD_BUFFERED, 0)) +#define IOCTL_IMDISK_REFERENCE_HANDLE ((ULONG) CTL_CODE(FILE_DEVICE_IMDISK, 0x804, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS)) +#define IOCTL_IMDISK_SET_DEVICE_FLAGS ((ULONG) CTL_CODE(FILE_DEVICE_IMDISK, 0x805, METHOD_BUFFERED, 0)) +#define IOCTL_IMDISK_REMOVE_DEVICE ((ULONG) CTL_CODE(FILE_DEVICE_IMDISK, 0x806, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS)) +#define IOCTL_IMDISK_IOCTL_PASS_THROUGH ((ULONG) CTL_CODE(FILE_DEVICE_IMDISK, 0x807, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS)) +#define IOCTL_IMDISK_FSCTL_PASS_THROUGH ((ULONG) CTL_CODE(FILE_DEVICE_IMDISK, 0x808, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS)) +#define IOCTL_IMDISK_GET_REFERENCED_HANDLE ((ULONG) CTL_CODE(FILE_DEVICE_IMDISK, 0x809, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS)) + +/// +/// Bit constants for the Flags field in IMDISK_CREATE_DATA +/// + +/// Read-only device +#define IMDISK_OPTION_RO 0x00000001 + +/// Check if flags specifies read-only +#define IMDISK_READONLY(x) ((ULONG)(x) & 0x00000001) + +/// Removable, hot-plug, device +#define IMDISK_OPTION_REMOVABLE 0x00000002 + +/// Check if flags specifies removable +#define IMDISK_REMOVABLE(x) ((ULONG)(x) & 0x00000002) + +/// Specifies that image file is created with sparse attribute. +#define IMDISK_OPTION_SPARSE_FILE 0x00000004 + +/// Check if flags specifies sparse +#define IMDISK_SPARSE_FILE(x) ((ULONG)(x) & 0x00000004) + +/// Swaps each byte pair in image file. +#define IMDISK_OPTION_BYTE_SWAP 0x00000008 + +/// Check if flags specifies byte swapping +#define IMDISK_BYTE_SWAP(x) ((ULONG)(x) & 0x00000008) + +/// Device type is virtual harddisk partition +#define IMDISK_DEVICE_TYPE_HD 0x00000010 +/// Device type is virtual floppy drive +#define IMDISK_DEVICE_TYPE_FD 0x00000020 +/// Device type is virtual CD/DVD-ROM drive +#define IMDISK_DEVICE_TYPE_CD 0x00000030 +/// Device type is unknown "raw" (for use with third-party client drivers) +#define IMDISK_DEVICE_TYPE_RAW 0x00000040 + +/// Extracts the IMDISK_DEVICE_TYPE_xxx from flags +#define IMDISK_DEVICE_TYPE(x) ((ULONG)(x) & 0x000000F0) + +/// Virtual disk is backed by image file +#define IMDISK_TYPE_FILE 0x00000100 +/// Virtual disk is backed by virtual memory +#define IMDISK_TYPE_VM 0x00000200 +/// Virtual disk is backed by proxy connection +#define IMDISK_TYPE_PROXY 0x00000300 + +/// Extracts the IMDISK_TYPE_xxx from flags +#define IMDISK_TYPE(x) ((ULONG)(x) & 0x00000F00) + +// Types with proxy mode + +/// Proxy connection is direct-type +#define IMDISK_PROXY_TYPE_DIRECT 0x00000000 +/// Proxy connection is over serial line +#define IMDISK_PROXY_TYPE_COMM 0x00001000 +/// Proxy connection is over TCP/IP +#define IMDISK_PROXY_TYPE_TCP 0x00002000 +/// Proxy connection uses shared memory +#define IMDISK_PROXY_TYPE_SHM 0x00003000 + +/// Extracts the IMDISK_PROXY_TYPE_xxx from flags +#define IMDISK_PROXY_TYPE(x) ((ULONG)(x) & 0x0000F000) + +// Types with file mode + +/// Serialized I/O to an image file, done in a worker thread +#define IMDISK_FILE_TYPE_QUEUED_IO 0x00000000 +/// Direct parallel I/O to AWEAlloc driver (physical RAM), done in request +/// thread +#define IMDISK_FILE_TYPE_AWEALLOC 0x00001000 +/// Direct parallel I/O to an image file, done in request thread +#define IMDISK_FILE_TYPE_PARALLEL_IO 0x00002000 + +/// Extracts the IMDISK_FILE_TYPE_xxx from flags +#define IMDISK_FILE_TYPE(x) ((ULONG)(x) & 0x0000F000) + +/// Flag set by write request dispatchers to indicated that virtual disk has +/// been since mounted +#define IMDISK_IMAGE_MODIFIED 0x00010000 + +/// This flag causes the driver to open image files in shared write mode even +/// if the image is opened for writing. This could be useful in some cases, +/// but could easily corrupt filesystems on image files if used incorrectly. +#define IMDISK_OPTION_SHARED_IMAGE 0x00020000 +/// Check if flags indicate shared write mode +#define IMDISK_SHARED_IMAGE(x) ((ULONG)(x) & 0x00020000) + +/// Macro to determine if flags specify either virtual memory (type vm) or +/// physical memory (type file with awealloc) virtual disk drive +#define IMDISK_IS_MEMORY_DRIVE(x) \ + ((IMDISK_TYPE(x) == IMDISK_TYPE_VM) || \ + ((IMDISK_TYPE(x) == IMDISK_TYPE_FILE) && \ + (IMDISK_FILE_TYPE(x) == IMDISK_FILE_TYPE_AWEALLOC))) + +/// Specify as device number to automatically select first free. +#define IMDISK_AUTO_DEVICE_NUMBER ((ULONG)-1) + +/** +Structure used by the IOCTL_IMDISK_CREATE_DEVICE and +IOCTL_IMDISK_QUERY_DEVICE calls and by the ImDiskQueryDevice function. +*/ +typedef struct _IMDISK_CREATE_DATA +{ + /// On create this can be set to IMDISK_AUTO_DEVICE_NUMBER + ULONG DeviceNumber; + /// Total size in bytes (in the Cylinders field) and virtual geometry. + DISK_GEOMETRY DiskGeometry; + /// The byte offset in the image file where the virtual disk begins. + LARGE_INTEGER ImageOffset; + /// Creation flags. Type of device and type of connection. + ULONG Flags; + /// Drive letter (if used, otherwise zero). + WCHAR DriveLetter; + /// Length in bytes of the FileName member. + USHORT FileNameLength; + /// Dynamically-sized member that specifies the image file name. + WCHAR FileName[1]; +} IMDISK_CREATE_DATA, *PIMDISK_CREATE_DATA; + +typedef struct _IMDISK_SET_DEVICE_FLAGS +{ + ULONG FlagsToChange; + ULONG FlagValues; +} IMDISK_SET_DEVICE_FLAGS, *PIMDISK_SET_DEVICE_FLAGS; + +#define IMDISK_API_NO_BROADCAST_NOTIFY 0x00000001 +#define IMDISK_API_FORCE_DISMOUNT 0x00000002 + +#pragma pack(push) +#pragma pack(1) +typedef struct _FAT_BPB +{ + USHORT BytesPerSector; + UCHAR SectorsPerCluster; + USHORT ReservedSectors; + UCHAR NumberOfFileAllocationTables; + USHORT NumberOfRootEntries; + USHORT NumberOfSectors; + UCHAR MediaDescriptor; + USHORT SectorsPerFileAllocationTable; + USHORT SectorsPerTrack; + USHORT NumberOfHeads; + union + { + struct + { + USHORT NumberOfHiddenSectors; + USHORT TotalNumberOfSectors; + } DOS_320; + struct + { + ULONG NumberOfHiddenSectors; + ULONG TotalNumberOfSectors; + } DOS_331; + }; +} FAT_BPB, *PFAT_BPB; + +typedef struct _FAT_VBR +{ + UCHAR JumpInstruction[3]; + CHAR OEMName[8]; + FAT_BPB BPB; + UCHAR FillData[512 - 3 - 8 - sizeof(FAT_BPB) - 1 - 2]; + UCHAR PhysicalDriveNumber; + UCHAR Signature[2]; +} FAT_VBR, *PFAT_VBR; +#pragma pack(pop) + +#ifdef WINAPI + +#ifdef __cplusplus +extern "C" { +#endif + + /** + Get behaviour flags for API. + */ + IMDISK_API ULONGLONG + WINAPI + ImDiskGetAPIFlags(); + + /** + Set behaviour flags for API. Returns previously defined flag field. + + Flags New flags value to set. + */ + IMDISK_API ULONGLONG + WINAPI + ImDiskSetAPIFlags(ULONGLONG Flags); + + /** + An interactive rundll32.exe-compatible function to show the Add New Virtual + Disk dialog box with a file name already filled in. It is used by the + Windows Explorer context menus. + + hWnd Specifies a window that will be the owner window of any + MessageBox:es or similar. + + hInst Ignored. + + lpszCmdLine An ANSI string specifying the image file to mount. + + nCmdShow Ignored. + */ + IMDISK_API void + WINAPI + RunDLL_MountFile(HWND hWnd, + HINSTANCE hInst, + LPSTR lpszCmdLine, + int nCmdShow); + + /** + An interactive rundll32.exe-compatible function to remove an existing ImDisk + virtual disk. If the filesystem on the device cannot be locked and + dismounted a MessageBox is displayed that asks the user if dismount should + be forced. + + hWnd Specifies a window that will be the owner window of any + MessageBox:es or similar. + + hInst Ignored. + + lpszCmdLine An ANSI string specifying the virtual disk to remove. This + can be on the form "F:" or "F:\" (without the quotes). + + nCmdShow Ignored. + */ + IMDISK_API void + WINAPI + RunDLL_RemoveDevice(HWND hWnd, + HINSTANCE hInst, + LPSTR lpszCmdLine, + int nCmdShow); + + /** + An interactive rundll32.exe-compatible function to save a virtual or + physical drive as an image file. If the filesystem on the device cannot be + locked and dismounted a MessageBox is displayed that asks the user if the + image saving should continue anyway. + + hWnd Specifies a window that will be the owner window of any + MessageBox:es or similar. + + hInst Ignored. + + lpszCmdLine An ANSI string specifying the disk to save. This can be on + the form "F:" or "F:\" (without the quotes). + + nCmdShow Ignored. + */ + IMDISK_API void + WINAPI + RunDLL_SaveImageFile(HWND hWnd, + HINSTANCE hInst, + LPSTR lpszCmdLine, + int nCmdShow); + + /** + This function displays a MessageBox dialog with a + FormatMessage-formatted message. + + hWndParent Parent window for the MessageBox call. + + uStyle Style for the MessageBox call. + + lpTitle Window title for the MessageBox call. + + lpMessage Format string to be used in call to FormatMessage followed + by field parameters. + */ + IMDISK_API BOOL + CDECL + ImDiskMsgBoxPrintF(IN HWND hWndParent OPTIONAL, + IN UINT uStyle, + IN LPCWSTR lpTitle, + IN LPCWSTR lpMessage, ...); + + /** + Synchronously flush Windows message queue to make GUI components responsive. + */ + IMDISK_API VOID + WINAPI + ImDiskFlushWindowMessages(HWND hWnd); + + /** + Used to get a string describing a partition type. + + PartitionType Partition type from partition table. + + Name Pointer to memory that receives a string describing the + partition type. + + NameSize Size of memory area pointed to by the Name parameter. + */ + IMDISK_API VOID + WINAPI + ImDiskGetPartitionTypeName(IN BYTE PartitionType, + IN OUT LPWSTR Name, + IN DWORD NameSize); + + /** + Returns the offset in bytes to actual disk image data for some known + "non-raw" image file formats with headers. Returns TRUE if file extension + is recognized and the known offset has been stored in the variable pointed + to by the Offset parameter. Otherwise the function returns FALSE and the + value pointed to by the Offset parameter is not changed. + + ImageFile Name of raw disk image file. This does not need to be a valid + path or filename, just the extension is used by this function. + + Offset Returned offset in bytes if function returns TRUE. + */ + IMDISK_API BOOL + WINAPI + ImDiskGetOffsetByFileExt(IN LPCWSTR ImageFile, + IN OUT PLARGE_INTEGER Offset); + + /** + Attempts to find partition information from a partition table for a raw + disk image file. If no master boot record is found this function returns + FALSE. Returns TRUE if a master boot record with a partition table is found + and values stored in the structures pointed to by the PartitionInformation + parameter. Otherwise the function returns FALSE. + + ImageFile Name of raw disk image file to examine. + + SectorSize Optional sector size used on disk if different from default + 512 bytes. + + Offset Optional offset in bytes to master boot record within file for + use with "non-raw" image files with headers before the actual + disk image data. + + PartitionInformation + Pointer to an array of eight PARTITION_INFORMATION structures + which will receive information from four recognized primary + partition entries followed by four recognized extended entries. + */ + IMDISK_API BOOL + WINAPI + ImDiskGetPartitionInformation(IN LPCWSTR ImageFile, + IN DWORD SectorSize OPTIONAL, + IN PLARGE_INTEGER Offset OPTIONAL, + IN OUT PPARTITION_INFORMATION + PartitionInformation); + + + /** + Prototype for raw disk reader function used with ImDisk***Indirect + functions. + + Handle Value that was passed as first parameter to + ImDiskGetPartitionInfoIndirect. + + Buffer Buffer where read data is to be stored. + + Offset Disk offset where read operation starts. + + NumberOfBytesToRead Number of bytes to read from disk. + + NumberOfBytesRead Pointer to DWORD size variable where function stores + number of bytes actually read into Buffer. This value + can be equal to or less than NumberOfBytesToRead + parameter. + */ + typedef BOOL(WINAPI *ImDiskReadFileProc)(IN HANDLE Handle, + IN OUT LPVOID Buffer, + IN LARGE_INTEGER Offset, + IN DWORD NumberOfBytesToRead, + IN OUT LPDWORD NumberOfBytesRead); + + /** + A device read function with ImDiskReadFileProc, which means that it can be + used when calling ImDiskGetPartitionInfoIndirect function. + + Handle Operating system file handle representing a file or device + opened for reading. + + Buffer Buffer where read data is to be stored. + + Offset Disk offset where read operation starts. + + NumberOfBytesToRead + Number of bytes to read from disk. + + NumberOfBytesRead + Pointer to DWORD size variable where function stores number of + bytes actually read into Buffer. This value can be equal to or + less than NumberOfBytesToRead parameter. + */ + IMDISK_API BOOL + WINAPI + ImDiskReadFileHandle(IN HANDLE Handle, + IN OUT LPVOID Buffer, + IN LARGE_INTEGER Offset, + IN DWORD NumberOfBytesToRead, + IN OUT LPDWORD NumberOfBytesRead); + + /** + Attempts to find partition information from a partition table for a disk + image through a supplied device reader function. + + If no master boot record is found this function returns FALSE. Returns TRUE + if a master boot record with a partition table is found and values stored in + the structures pointed to by the PartitionInformation parameter. Otherwise + the function returns FALSE. + + Handle Value that is passed as first parameter to ReadFileProc. + + ReadFileProc Procedure of type ImDiskReadFileProc that is called to read raw + disk image. + + SectorSize Optional sector size used on disk if different from default + 512 bytes. + + Offset Optional offset in bytes to master boot record within file for + use with "non-raw" image files with headers before the actual + disk image data. + + PartitionInformation + Pointer to an array of eight PARTITION_INFORMATION structures + which will receive information from four recognized primary + partition entries followed by four recognized extended entries. + */ + IMDISK_API BOOL + WINAPI + ImDiskGetPartitionInfoIndirect(IN HANDLE Handle, + IN ImDiskReadFileProc ReadFileProc, + IN DWORD SectorSize OPTIONAL, + IN PLARGE_INTEGER Offset OPTIONAL, + IN OUT PPARTITION_INFORMATION PartitionInfo); + + /** + Finds out if image file contains an ISO9660 filesystem. + + ImageFile Name of disk image file to examine. + + Offset Optional offset in bytes to where raw disk data begins, for use + with "non-raw" image files with headers before the actual disk + image data. + */ + IMDISK_API BOOL + WINAPI + ImDiskImageContainsISOFS(IN LPCWSTR ImageFile, + IN PLARGE_INTEGER Offset OPTIONAL); + + /** + Finds out if image file contains an ISO9660 filesystem, through a supplied + device reader function. + + Handle Value that is passed as first parameter to ReadFileProc. + + ReadFileProc Procedure of type ImDiskReadFileProc that is called to read raw + disk image. + + Offset Optional offset in bytes to where raw disk data begins, for use + with "non-raw" image files with headers before the actual disk + image data. + */ + IMDISK_API BOOL + WINAPI + ImDiskImageContainsISOFSIndirect(IN HANDLE Handle, + IN ImDiskReadFileProc ReadFileProc, + IN PLARGE_INTEGER Offset OPTIONAL); + + /** + Starts a Win32 service or loads a kernel module or driver. + + ServiceName Key name of the service or driver. + */ + IMDISK_API BOOL + WINAPI + ImDiskStartService(IN LPWSTR ServiceName); + + /** + An easy way to turn an empty NTFS directory to a reparse point that redirects + requests to a mounted device. Acts quite like mount points or symbolic links + in *nix. If MountPoint specifies a character followed by a colon, a drive + letter is instead created to point to Target. + + MountPoint Path to empty directory on an NTFS volume, or a drive letter + followed by a colon. + + Target Target device path on kernel object namespace form, e.g. + \Device\ImDisk2 or similar. + */ + IMDISK_API BOOL + WINAPI + ImDiskCreateMountPoint(IN LPCWSTR MountPoint, + IN LPCWSTR Target); + + /** + Restores a reparse point to be an ordinary empty directory, or removes a + drive letter mount point. When removing a drive letter mount point, this + function notifies shell components that drive letter is gone unless API + flags are set to turn off shell notifications. + + MountPoint Path to a reparse point on an NTFS volume, or a drive letter + followed by a colon to remove a drive letter mount point. + */ + IMDISK_API BOOL + WINAPI + ImDiskRemoveMountPoint(IN LPCWSTR MountPoint); + + /** + Opens a device object in the kernel object namespace. + + FileName Full kernel object namespace path to the object to open, e.g. + \Device\ImDisk2 or similar. + + AccessMode Access mode to request. + */ + IMDISK_API HANDLE + WINAPI + ImDiskOpenDeviceByName(IN PUNICODE_STRING FileName, + IN DWORD AccessMode); + + /** + Opens an ImDisk device by the device number. + + DeviceNumber Number of the ImDisk device to open. + + AccessMode Access mode to request. + */ + IMDISK_API HANDLE + WINAPI + ImDiskOpenDeviceByNumber(IN DWORD DeviceNumber, + IN DWORD AccessMode); + + /** + Opens the device a junction/mount-point type reparse point is pointing to. + + MountPoint Path to the reparse point on an NTFS volume. + + AccessMode Access mode to request to the target device. + */ + IMDISK_API HANDLE + WINAPI + ImDiskOpenDeviceByMountPoint(IN LPCWSTR MountPoint, + IN DWORD AccessMode); + + /** + Check that the user-mode library and kernel-mode driver version matches for + an open ImDisk created device object. + + DeviceHandle Handle to an open ImDisk virtual disk or control device. + */ + IMDISK_API BOOL + WINAPI + ImDiskCheckDriverVersion(IN HANDLE DeviceHandle); + + /** + Retrieves the version numbers of the user-mode API library and the kernel- + mode driver. + */ + IMDISK_API BOOL + WINAPI + ImDiskGetVersion(IN OUT PULONG LibraryVersion OPTIONAL, + IN OUT PULONG DriverVersion OPTIONAL); + + /** + Returns the first free drive letter in the range D-Z. + */ + IMDISK_API WCHAR + WINAPI + ImDiskFindFreeDriveLetter(); + + /** + Returns a bit-field representing ImDisk devices. Bit 0 represents device 0, + bit 1 represents device 1 and so on. A bit is 1 if the device exists or 0 if + the device number is free. + + Compatibility notice: + This function is exported for compatibility with ImDisk versions before + 1.7.0. Since that version, drives can have device numbers above 63. This + function cannot return such device numbers, so in case any drive with device + number above 63 exist when this function is called, it returns a value + filled with all ones ((ULONGLONG)-1). + + Use ImDiskGetDeviceListEx function with newer versions of ImDisk. + */ + IMDISK_API ULONGLONG + WINAPI + ImDiskGetDeviceList(); + + /** + Builds a list of currently existing ImDisk virtual disks. + + ListLength Set this parameter to number of ULONG element that can be + store at the location pointed to by DeviceList parameter. + This parameter must be at least 3 for this function to work + correctly. + + DeviceList Pointer to memory location where one ULONG, containing a + device number, will be stored for each currently existing + ImDisk device. First element in list is used to store number + of devices. + + Upon return, first element in DeviceList will contain number of currently + existing ImDisk virtual disks. If DeviceList is too small to contain all + items as indicated by ListLength parameter, number of existing devices will + be stored at DeviceList location, but no further items will be stored. + + If an error occurs, this function returns FALSE and GetLastError + will return an error code. If successful, the function returns TRUE and + first element at location pointed to by DeviceList will contain number of + devices currently on the system, i.e. number of elements following the first + one in DeviceList. + + If DeviceList buffer is too small, the function returns FALSE and + GetLastError returns ERROR_MORE_DATA. In that case, only number of + existing devices will be stored at location pointed to by DeviceList + parameter. That value, plus one for the first length element, indicates how + large the buffer needs to be to successfully store all items. + */ + IMDISK_API BOOL + WINAPI + ImDiskGetDeviceListEx(IN ULONG ListLength, + OUT PULONG DeviceList); + + /** + This function sends an IOCTL_IMDISK_QUERY_DEVICE control code to an existing + device and returns information about the device in an IMDISK_CREATE_DATA + structure. + + DeviceNumber Number of the ImDisk device to query. + + CreateData Pointer to a sufficiently large IMDISK_CREATE_DATA + structure to receive all data including the image file name + where applicable. + + CreateDataSize The size in bytes of the memory the CreateData parameter + points to. The function call will fail if the memory is not + large enough to hold the entire IMDISK_CREATE_DATA + structure. + */ + IMDISK_API BOOL + WINAPI + ImDiskQueryDevice(IN DWORD DeviceNumber, + IN OUT PIMDISK_CREATE_DATA CreateData, + IN ULONG CreateDataSize); + + /** + This function creates a new ImDisk virtual disk device. + + hWndStatusText A handle to a window that can display status message text. + The function will send WM_SETTEXT messages to this window. + If this parameter is NULL no WM_SETTEXT messages are sent + and the function acts non-interactive. + + DiskGeometry The virtual geometry of the new virtual disk. Note that the + Cylinders member does not specify the number of Cylinders + but the total size in bytes of the new virtual disk. The + actual number of cylinders are then automatically + calculated and rounded down if necessary. + + The Cylinders member can be zero if the device is backed by + an image file or a proxy device, but not if it is virtual + memory only device. + + All or some of the other members of this structure can be + zero in which case they are automatically filled in with + most reasonable values by the driver. + + Flags Bitwise or-ed combination of one of the IMDISK_TYPE_xxx + flags, one of the IMDISK_DEVICE_TYPE_xxx flags and any + number of IMDISK_OPTION_xxx flags. The flags can often be + left zero and left to the driver to automatically select. + For example, if a virtual disk size is specified to 1440 KB + and an image file name is not specified, the driver + automatically selects IMDISK_TYPE_VM|IMDISK_DEVICE_TYPE_FD + for this parameter. + + FileName Name of disk image file. In case IMDISK_TYPE_VM is + specified in the Flags parameter, this file will be loaded + into the virtual memory-backed disk when created. + + NativePath Set to TRUE if the FileName parameter specifies an NT + native path, such as \??\C:\imagefile.img or FALSE if it + specifies a Win32/DOS-style path such as C:\imagefile.img. + + MountPoint Drive letter to assign to the new virtual device. It can be + specified on the form F: or F:\. It can also specify an empty directory + on another NTFS volume. + */ + IMDISK_API BOOL + WINAPI + ImDiskCreateDevice(IN HWND hWndStatusText OPTIONAL, + IN OUT PDISK_GEOMETRY DiskGeometry OPTIONAL, + IN PLARGE_INTEGER ImageOffset OPTIONAL, + IN DWORD Flags OPTIONAL, + IN LPCWSTR FileName OPTIONAL, + IN BOOL NativePath, + IN LPWSTR MountPoint OPTIONAL); + + /** + This function creates a new ImDisk virtual disk device. + + hWndStatusText A handle to a window that can display status message text. + The function will send WM_SETTEXT messages to this window. + If this parameter is NULL no WM_SETTEXT messages are sent + and the function acts non-interactive. + + DeviceNumber In: Device number for device to create. Device number must + not be in use by an existing virtual disk. For automatic + allocation of device number, use IMDISK_AUTO_DEVICE_NUMBER + constant or specify a NULL pointer. + + Out: If DeviceNumber parameter is not NULL, device number + for created device is returned in DWORD variable pointed to. + + DiskGeometry The virtual geometry of the new virtual disk. Note that the + Cylinders member does not specify the number of Cylinders + but the total size in bytes of the new virtual disk. The + actual number of cylinders are then automatically + calculated and rounded down if necessary. + + The Cylinders member can be zero if the device is backed by + an image file or a proxy device, but not if it is virtual + memory only device. + + All or some of the other members of this structure can be + zero in which case they are automatically filled in with + most reasonable values by the driver. + + Flags Bitwise or-ed combination of one of the IMDISK_TYPE_xxx + flags, one of the IMDISK_DEVICE_TYPE_xxx flags and any + number of IMDISK_OPTION_xxx flags. The flags can often be + left zero and left to the driver to automatically select. + For example, if a virtual disk size is specified to 1440 KB + and an image file name is not specified, the driver + automatically selects IMDISK_TYPE_VM|IMDISK_DEVICE_TYPE_FD + for this parameter. + + FileName Name of disk image file. In case IMDISK_TYPE_VM is + specified in the Flags parameter, this file will be loaded + into the virtual memory-backed disk when created. + + NativePath Set to TRUE if the FileName parameter specifies an NT + native path, such as \??\C:\imagefile.img or FALSE if it + specifies a Win32/DOS-style path such as C:\imagefile.img. + + MountPoint Drive letter to assign to the new virtual device. It can + be specified on the form F: or F:\. It can also specify an empty directory + on another NTFS volume. + */ + IMDISK_API BOOL + WINAPI + ImDiskCreateDeviceEx(IN HWND hWndStatusText OPTIONAL, + IN OUT LPDWORD DeviceNumber OPTIONAL, + IN OUT PDISK_GEOMETRY DiskGeometry OPTIONAL, + IN PLARGE_INTEGER ImageOffset OPTIONAL, + IN DWORD Flags OPTIONAL, + IN LPCWSTR FileName OPTIONAL, + IN BOOL NativePath, + IN LPWSTR MountPoint OPTIONAL); + + /** + This function removes (unmounts) an existing ImDisk virtual disk device. + + hWndStatusText A handle to a window that can display status message text. + The function will send WM_SETTEXT messages to this window. + If this parameter is NULL no WM_SETTEXT messages are sent + and the function acts non-interactive. + + DeviceNumber Number of the ImDisk device to remove. This parameter is + only used if MountPoint parameter is null. + + MountPoint Drive letter of the device to remove. It can be specified + on the form F: or F:\. + */ + IMDISK_API BOOL + WINAPI + ImDiskRemoveDevice(IN HWND hWndStatusText OPTIONAL, + IN DWORD DeviceNumber OPTIONAL, + IN LPCWSTR MountPoint OPTIONAL); + + /** + This function forcefully removes (unmounts) an existing ImDisk virtual disk + device. Any unsaved data will be lost. + + Device Handle to open device. If not NULL, it is used to query + device number to find out which device to remove. If this + parameter is NULL the DeviceNumber parameter is used + instead. + + DeviceNumber Number of the ImDisk device to remove. This parameter is + only used if Device parameter is NULL. + */ + IMDISK_API BOOL + WINAPI + ImDiskForceRemoveDevice(IN HANDLE Device OPTIONAL, + IN DWORD DeviceNumber OPTIONAL); + + /** + This function changes the device characteristics of an existing ImDisk + virtual disk device. + + hWndStatusText A handle to a window that can display status message text. + The function will send WM_SETTEXT messages to this window. + If this parameter is NULL no WM_SETTEXT messages are sent + and the function acts non-interactive. + + DeviceNumber Number of the ImDisk device to change. This parameter is + only used if MountPoint parameter is null. + + MountPoint Drive letter of the device to change. It can be specified + on the form F: or F:\. + + FlagsToChange A bit-field specifying which flags to edit. The flags are + the same as the option flags in the Flags parameter used + when a new virtual disk is created. Only flags set in this + parameter are changed to the corresponding flag value in the + Flags parameter. + + Flags New values for the flags specified by the FlagsToChange + parameter. + */ + IMDISK_API BOOL + WINAPI + ImDiskChangeFlags(HWND hWndStatusText OPTIONAL, + DWORD DeviceNumber OPTIONAL, + LPCWSTR MountPoint OPTIONAL, + DWORD FlagsToChange, + DWORD Flags); + + /** + This function extends the size of an existing ImDisk virtual disk device. + + hWndStatusText A handle to a window that can display status message text. + The function will send WM_SETTEXT messages to this window. + If this parameter is NULL no WM_SETTEXT messages are sent + and the function acts non-interactive. + + DeviceNumber Number of the ImDisk device to extend. + + ExtendSize A pointer to a LARGE_INTEGER structure that specifies the + number of bytes to extend the device. + */ + IMDISK_API BOOL + WINAPI + ImDiskExtendDevice(IN HWND hWndStatusText OPTIONAL, + IN DWORD DeviceNumber, + IN CONST PLARGE_INTEGER ExtendSize); + + /** + This function saves the contents of a device to an image file. + + DeviceHandle Handle to a device for which the contents are to be saved to + an image file. + + The handle must be opened for reading, may be + opened for sequential scan and/or without intermediate + buffering but cannot be opened for overlapped operation. + Please note that a call to this function will turn on + FSCTL_ALLOW_EXTENDED_DASD_IO on for this handle. + + FileHandle Handle to an image file opened for writing. The handle + can be opened for operation without intermediate buffering + but performance is usually better if the handle is opened + with intermediate buffering. The handle cannot be opened for + overlapped operation. + + BufferSize I/O buffer size to use when reading source disk. This + parameter is optional, if it is zero the buffer size to use + will automatically chosen. + + CancelFlag Optional pointer to a BOOL value. If this BOOL value is set + to TRUE during the function call the operation is cancelled, + the function returns FALSE and GetLastError will return + ERROR_CANCELLED. If this parameter is non-null the function + will also dispatch window messages for the current thread + between each I/O operation. + */ + IMDISK_API BOOL + WINAPI + ImDiskSaveImageFile(IN HANDLE DeviceHandle, + IN HANDLE FileHandle, + IN DWORD BufferSize OPTIONAL, + IN LPBOOL CancelFlag OPTIONAL); + + /** + This function gets the size of a disk volume. + + Handle Handle to a disk volume device. + + Size Pointer to a 64 bit variable that upon successful completion + receives disk volume size as a signed integer. + */ + IMDISK_API BOOL + WINAPI + ImDiskGetVolumeSize(IN HANDLE Handle, + IN OUT PLONGLONG Size); + + /** + Reads formatted geometry for a volume by parsing BPB, BIOS Parameter Block, + from volume boot record into a DISK_GEOMETRY structure. + + If no boot record signature is found, this function returns FALSE. + + ImageFile Path to a volume image file or a device path to a disk volume, + such as \\.\A: or \\.\C:. + + Offset Optional offset in bytes to volume boot record within file for + use with "non-raw" volume image files. This parameter can be + used to for example skip over headers for specific disk image + formats, or to skip over master boot record in a disk image + file that contains a complete raw disk image and not only a + single volume. + + DiskGeometry Pointer to DISK_GEOMETRY structure that receives information + about formatted geometry. This function zeros the Cylinders + member. + */ + IMDISK_API BOOL + WINAPI + ImDiskGetFormattedGeometry(IN LPCWSTR ImageFile, + IN PLARGE_INTEGER Offset OPTIONAL, + IN OUT PDISK_GEOMETRY DiskGeometry); + + /** + Reads formatted geometry for a volume by parsing BPB, BIOS Parameter Block, + from volume boot record into a DISK_GEOMETRY structure. + + If no boot record signature is found, this function returns FALSE. + + Handle Value that is passed as first parameter to ReadFileProc. + + ReadFileProc Procedure of type ImDiskReadFileProc that is called to read + disk volume. + + Offset Optional offset in bytes to volume boot record within file for + use with "non-raw" volume image files. This parameter can be + used to for example skip over headers for specific disk image + formats, or to skip over master boot record in a disk image + file that contains a complete raw disk image and not only a + single volume. + + DiskGeometry Pointer to DISK_GEOMETRY structure that receives information + about formatted geometry. This function zeros the Cylinders + member. + */ + IMDISK_API BOOL + WINAPI + ImDiskGetFormattedGeometryIndirect(IN HANDLE Handle, + IN ImDiskReadFileProc ReadFileProc, + IN PLARGE_INTEGER Offset OPTIONAL, + IN OUT PDISK_GEOMETRY DiskGeometry); + + /** + This function builds a Master Boot Record, MBR, in memory. The MBR will + contain a default Initial Program Loader, IPL, which could be used to boot + an operating system partition when the MBR is written to a disk. + + DiskGeometry Pointer to a DISK_GEOMETRY or DISK_GEOMETRY_EX structure + that contains information about logical geometry of the + disk. + + This function only uses the BytesPerSector, SectorsPerTrack + and TracksPerCylinder members. + + This parameter can be NULL if NumberOfParts parameter is + zero. + + PartitionInfo Pointer to an array of up to four PARTITION_INFORMATION + structures containing information about partitions to store + in MBR partition table. + + This function only uses the StartingOffset, PartitionLength, + BootIndicator and PartitionType members. + + This parameter can be NULL if NumberOfParts parameter is + zero. + + NumberOfParts Number of PARTITION_INFORMATION structures in array that + PartitionInfo parameter points to. + + If this parameter is zero, DiskGeometry and PartitionInfo + parameters are ignored and can be NULL. In that case MBR + will contain an empty partition table when this function + returns. + + MBR Pointer to memory buffer of at least 512 bytes where MBR + will be built. + + MBRSize Size of buffer pointed to by MBR parameter. This parameter + must be at least 512. + */ + IMDISK_API BOOL + WINAPI + ImDiskBuildMBR(IN PDISK_GEOMETRY DiskGeometry OPTIONAL, + IN PPARTITION_INFORMATION PartitionInfo OPTIONAL, + IN BYTE NumberOfParts OPTIONAL, + IN OUT LPBYTE MBR, + IN DWORD_PTR MBRSize); + + /** + This function converts a CHS disk address to LBA format. + + DiskGeometry Pointer to a DISK_GEOMETRY or DISK_GEOMETRY_EX structure + that contains information about logical geometry of the + disk. This function only uses the SectorsPerTrack and + TracksPerCylinder members. + + CHS Pointer to CHS disk address in three-byte partition table + style format. + */ + IMDISK_API DWORD + WINAPI + ImDiskConvertCHSToLBA(IN PDISK_GEOMETRY DiskGeometry, + IN LPBYTE CHS); + + /** + This function converts an LBA disk address to three-byte partition style CHS + format. The three bytes are returned in the three lower bytes of a DWORD. + + DiskGeometry Pointer to a DISK_GEOMETRY or DISK_GEOMETRY_EX structure + that contains information about logical geometry of the + disk. This function only uses the SectorsPerTrack and + TracksPerCylinder members. + + LBA LBA disk address. + */ + IMDISK_API DWORD + WINAPI + ImDiskConvertLBAToCHS(IN PDISK_GEOMETRY DiskGeometry, + IN DWORD LBA); + + /** + This function adjusts size of a saved image file. If file size is less than + requested disk size, the size will be left unchanged with return value FALSE + and GetLastError will return ERROR_DISK_OPERATION_FAILED. + + FileHandle Handle to file where disk image has been saved. + + FileSize Size of original disk which image file should be adjusted + to. + */ + IMDISK_API BOOL + WINAPI + ImDiskAdjustImageFileSize(IN HANDLE FileHandle, + IN PLARGE_INTEGER FileSize); + + /** + This function converts a native NT-style path to a Win32 DOS-style path. The + path string is converted in-place and the start address is adjusted to skip + over native directories such as \??\. Because of this, the Path parameter is + a pointer to a pointer to a string so that the pointer can be adjusted to + the new start address. + + Path Pointer to pointer to Path string in native NT-style format. + Upon return the pointed address will contain the start + address of the Win32 DOS-style path within the original + buffer. + */ + IMDISK_API VOID + WINAPI + ImDiskNativePathToWin32(IN OUT LPWSTR *Path); + + /** + This function saves the contents of a device to an image file. This is a + user-interactive function that displays dialog boxes where user can select + image file and other options. + + DeviceHandle Handle to a device for which the contents are to be saved to + an image file. + + The handle must be opened for reading, may be + opened for sequential scan and/or without intermediate + buffering but cannot be opened for overlapped operation. + Please note that a call to this function will turn on + FSCTL_ALLOW_EXTENDED_DASD_IO on for this handle. + + WindowHandle Handle to existing window that will be parent to dialog + boxes etc. + + BufferSize I/O buffer size to use when reading source disk. This + parameter is optional, if it is zero the buffer size to use + will automatically chosen. + + IsCdRomType If this parameter is TRUE and the source device type cannot + be automatically determined this function will ask user for + a .iso suffixed image file name. + */ + IMDISK_API VOID + WINAPI + ImDiskSaveImageFileInteractive(IN HANDLE DeviceHandle, + IN HWND WindowHandle OPTIONAL, + IN DWORD BufferSize OPTIONAL, + IN BOOL IsCdRomType OPTIONAL); + + /* + Opens or creates a global synchronization event. This event is shared with + ImDisk driver and will be pulsed when an ImDisk device is created, removed + or have settings changed in some other way. + + This is particularly useful for user interface components that need to be + notified when device lists and similar need to be updated. + + If successful, this function returns a handle to an event that can be used + in call to system wait functions, such as WaitForSingleObject. When the + handle is not needed, it must be closed by calling CloseHandle. + + If the function fails, it returns NULL and GetLastError will return a + system error code that further explains the error. + + InheritHandle Specifies whether or not the returned handle will be + inherited by child processes. + + */ + IMDISK_API HANDLE + WINAPI + ImDiskOpenRefreshEvent(BOOL InheritHandle); + + /* + Adds registry settings for creating a virtual disk at system startup (or + when driver is loaded). + + This function returns TRUE if successful, FALSE otherwise. If FALSE is + returned, GetLastError could be used to get actual error code. + + CreateData Pointer to IMDISK_CREATE_DATA structure that contains + device creation settings to save. + + */ + IMDISK_API BOOL + WINAPI + ImDiskSaveRegistrySettings(PIMDISK_CREATE_DATA CreateData); + + /* + Remove registry settings for creating a virtual disk at system startup (or + when driver is loaded). + + This function returns TRUE if successful, FALSE otherwise. If FALSE is + returned, GetLastError could be used to get actual error code. + + DeviceNumber Device number specified in registry settings. + */ + IMDISK_API BOOL + WINAPI + ImDiskRemoveRegistrySettings(DWORD DeviceNumber); + + /* + Retrieves number of auto-loading devices at system startup, or when driver + is loaded. This is the value of the LoadDevices registry value for + imdisk.sys driver. + + This function returns TRUE if successful, FALSE otherwise. If FALSE is + returned, GetLastError could be used to get actual error code. + + LoadDevicesValue + Pointer to variable that receives the value. + */ + IMDISK_API BOOL + WINAPI + ImDiskGetRegistryAutoLoadDevices(LPDWORD LoadDevicesValue); + + /* + Notify Explorer and other shell components that a new drive letter has + been created. Called automatically by device creation after creating a + drive letter. If no drive letter was created by a device creation routine + or if API flags was set to turn off shell notification during device + creation, this function can be called manually later. + + Note that calling this function has no effect if API flags are set to + turn off shell notifications, or if supplied drive letter path does not + specify an A-Z drive letter. + + This function returns TRUE if successful, FALSE otherwise. If FALSE is + returned, GetLastError could be used to get actual error code. + + hWnd + Window handle to use as parent handle for any message boxes. If this + parameter is NULL, no message boxes are displayed. + + DriveLetterPath + Drive letter path in one of formats A:\ or A:. + */ + IMDISK_API BOOL + WINAPI + ImDiskNotifyShellDriveLetter(HWND hWnd, + LPWSTR DriveLetterPath); + + /* + Notify Explorer and other shell components that a drive is about to be + removed. + + hWnd + Window handle to use as parent handle for any message boxes. If this + parameter is NULL, no message boxes are displayed. + + DriveLetter + Drive letter. + */ + IMDISK_API BOOL + WINAPI + ImDiskNotifyRemovePending(HWND hWnd, + WCHAR DriveLetter); + + IMDISK_API LPWSTR + CDECL + ImDiskAllocPrintF(LPCWSTR lpMessage, ...); + +#ifdef __cplusplus +} +#endif + +#endif + +#endif // _INC_IMDISK_ diff --git a/external/imdisk/imdiskver.h b/external/imdisk/imdiskver.h new file mode 100644 index 000000000..8c50dc543 --- /dev/null +++ b/external/imdisk/imdiskver.h @@ -0,0 +1,6 @@ +#define IMDISK_RC_VERSION_STR "2.0.9" +#define IMDISK_MAJOR_VERSION 2 +#define IMDISK_MINOR_VERSION 0 +#define IMDISK_MINOR_LOW_VERSION 9 + +#define IMDISK_RC_VERSION_FLD IMDISK_MAJOR_VERSION,IMDISK_MINOR_VERSION,IMDISK_MINOR_LOW_VERSION diff --git a/external/imdisk/imdproxy.h b/external/imdisk/imdproxy.h new file mode 100644 index 000000000..6d1a9291b --- /dev/null +++ b/external/imdisk/imdproxy.h @@ -0,0 +1,131 @@ +/* +ImDisk Proxy Services. + +Copyright (C) 2005-2007 Olof Lagerkvist. + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. +*/ + +#ifndef _INC_IMDPROXY_ +#define _INC_IMDPROXY_ + +#if !defined(_WIN32) && !defined(_NTDDK_) +typedef int32_t LONG; +typedef uint32_t ULONG; +typedef int64_t LONGLONG; +typedef uint64_t ULONGLONG; +typedef u_short WCHAR; +#endif + +#define IMDPROXY_SVC L"ImDskSvc" +#define IMDPROXY_SVC_PIPE_DOSDEV_NAME L"\\\\.\\PIPE\\" IMDPROXY_SVC +#define IMDPROXY_SVC_PIPE_NATIVE_NAME L"\\Device\\NamedPipe\\" IMDPROXY_SVC + +#define IMDPROXY_FLAG_RO 0x01 +#define IMDPROXY_FLAG_SUPPORTS_UNMAP 0x02 +#define IMDPROXY_FLAG_SUPPORTS_ZERO 0x04 + +typedef enum _IMDPROXY_REQ +{ + IMDPROXY_REQ_NULL, + IMDPROXY_REQ_INFO, + IMDPROXY_REQ_READ, + IMDPROXY_REQ_WRITE, + IMDPROXY_REQ_CONNECT, + IMDPROXY_REQ_CLOSE, + IMDPROXY_REQ_UNMAP, + IMDPROXY_REQ_ZERO +} IMDPROXY_REQ, *PIMDPROXY_REQ; + +typedef struct _IMDPROXY_CONNECT_REQ +{ + ULONGLONG request_code; + ULONGLONG flags; + ULONGLONG length; +} IMDPROXY_CONNECT_REQ, *PIMDPROXY_CONNECT_REQ; + +typedef struct _IMDPROXY_CONNECT_RESP +{ + ULONGLONG error_code; + ULONGLONG object_ptr; +} IMDPROXY_CONNECT_RESP, *PIMDPROXY_CONNECT_RESP; + +typedef struct _IMDPROXY_INFO_RESP +{ + ULONGLONG file_size; + ULONGLONG req_alignment; + ULONGLONG flags; +} IMDPROXY_INFO_RESP, *PIMDPROXY_INFO_RESP; + +typedef struct _IMDPROXY_READ_REQ +{ + ULONGLONG request_code; + ULONGLONG offset; + ULONGLONG length; +} IMDPROXY_READ_REQ, *PIMDPROXY_READ_REQ; + +typedef struct _IMDPROXY_READ_RESP +{ + ULONGLONG errorno; + ULONGLONG length; +} IMDPROXY_READ_RESP, *PIMDPROXY_READ_RESP; + +typedef struct _IMDPROXY_WRITE_REQ +{ + ULONGLONG request_code; + ULONGLONG offset; + ULONGLONG length; +} IMDPROXY_WRITE_REQ, *PIMDPROXY_WRITE_REQ; + +typedef struct _IMDPROXY_WRITE_RESP +{ + ULONGLONG errorno; + ULONGLONG length; +} IMDPROXY_WRITE_RESP, *PIMDPROXY_WRITE_RESP; + +typedef struct _IMDPROXY_UNMAP_REQ +{ + ULONGLONG request_code; + ULONGLONG length; +} IMDPROXY_UNMAP_REQ, *PIMDPROXY_UNMAP_REQ; + +typedef struct _IMDPROXY_UNMAP_RESP +{ + ULONGLONG errorno; +} IMDPROXY_UNMAP_RESP, *PIMDPROXY_UNMAP_RESP; + +typedef struct _IMDPROXY_ZERO_REQ +{ + ULONGLONG request_code; + ULONGLONG length; +} IMDPROXY_ZERO_REQ, *PIMDPROXY_ZERO_REQ; + +typedef struct _IMDPROXY_ZERO_RESP +{ + ULONGLONG errorno; +} IMDPROXY_ZERO_RESP, *PIMDPROXY_ZERO_RESP; + +// For shared memory proxy communication only. Offset to data area in +// shared memory. +#define IMDPROXY_HEADER_SIZE 4096 + +#endif // _INC_IMDPROXY_ diff --git a/file_linux.cpp b/file_linux.cpp index 1ff361e58..33aab846b 100644 --- a/file_linux.cpp +++ b/file_linux.cpp @@ -257,7 +257,7 @@ std::string File::Read(_u32 tr, bool *has_error) { std::string ret; ret.resize(tr); - _u32 gc=Read((char*)ret.c_str(), tr, has_error); + _u32 gc=Read(&ret[0], tr, has_error); if( gcsetCompressionSettings(comp_params); } CClientThread::~CClientThread() @@ -212,7 +215,7 @@ void CClientThread::operator()(void) { if (next_chunks.front().pipe_file_user == NULL) { - FileServ::decrShareActive(next_chunks.front().s_filename); + FileServ::decrShareActive(next_chunks.front().s_filename, next_chunks.front().share_active_gen); Server->destroy(next_chunks.front().update_file); } delete next_chunks.front().pipe_file_user; @@ -2008,7 +2011,7 @@ bool CClientThread::GetFileBlockdiff(CRData *data, bool with_metadata) hFile=INVALID_HANDLE_VALUE; - scoped_share_active.release(); + chunk.share_active_gen = scoped_share_active.release(); queueChunk(chunk); diff --git a/fileservplugin/CClientThread.h b/fileservplugin/CClientThread.h index 78c29507f..3730639fc 100644 --- a/fileservplugin/CClientThread.h +++ b/fileservplugin/CClientThread.h @@ -73,6 +73,7 @@ struct SChunk bool with_sparse; std::string s_filename; IFileServ::CbtHashFileInfo cbt_hash_file_info; + size_t share_active_gen; }; struct SLPData diff --git a/fileservplugin/CTCPFileServ.cpp b/fileservplugin/CTCPFileServ.cpp index be11b57b6..43fd2b12c 100644 --- a/fileservplugin/CTCPFileServ.cpp +++ b/fileservplugin/CTCPFileServ.cpp @@ -293,7 +293,7 @@ bool CTCPFileServ::TcpStep(void) SOCKET accept_socket = conn[s].fd; #endif SOCKET ns; - if (accept_socket == mSocketv6) + if (accept_socket == mSocket) { sockaddr_in naddr; socklen_t addrsize = sizeof(naddr); diff --git a/fileservplugin/CUDPThread.cpp b/fileservplugin/CUDPThread.cpp index 4d3055d75..b41a4aafe 100644 --- a/fileservplugin/CUDPThread.cpp +++ b/fileservplugin/CUDPThread.cpp @@ -49,7 +49,7 @@ namespace std::string mac_get_serial() { char buf[4096]; - FILE* fd = popen("system_profiler SPHardwareDataType | grep \"Serial Number\"", "r"); + FILE* fd = popen("/usr/sbin/system_profiler SPHardwareDataType | grep \"Serial Number\"", "r"); std::string serial; if (fd != NULL) { @@ -57,6 +57,7 @@ std::string mac_get_serial() { serial = trim(getafter(":", buf)); } + pclose(fd); } if (!serial.empty()) @@ -76,17 +77,17 @@ std::string mac_get_serial() std::string getSystemServerName(bool use_fqdn) { - char hostname[MAX_PATH]; #ifdef __APPLE__ //TODO: Fix FQDN for Apple while (true) { char hostname_appl[MAX_PATH + 15]; - FILE* fd = popen("system_profiler SPSoftwareDataType | grep \"Computer Name: \"", "r"); + FILE* fd = popen("/usr/sbin/system_profiler SPSoftwareDataType | grep \"Computer Name: \"", "r"); if (fd != NULL) { if (fgets(hostname_appl, sizeof(hostname_appl), fd) != NULL) { + pclose(fd); std::string chostname = getafter("Computer Name: ", trim(hostname_appl)); if (chostname.empty()) { @@ -107,14 +108,17 @@ std::string getSystemServerName(bool use_fqdn) return chostname + "-" + mac_add; } - pclose(fd); + else + pclose(fd); } else { Server->wait(100); } } -#endif +#else + + char hostname[MAX_PATH]; _i32 rc=gethostname(hostname, MAX_PATH); @@ -154,6 +158,7 @@ std::string getSystemServerName(bool use_fqdn) } return ret; +#endif } bool CUDPThread::hasError(void) diff --git a/fileservplugin/ChunkSendThread.cpp b/fileservplugin/ChunkSendThread.cpp index 0fab64f20..78aaad141 100644 --- a/fileservplugin/ChunkSendThread.cpp +++ b/fileservplugin/ChunkSendThread.cpp @@ -92,7 +92,7 @@ void ChunkSendThread::operator()(void) Server->Log("Closing file (free) " + file->getFilename(), LL_DEBUG); Server->destroy(file); assert(!s_filename.empty()); - FileServ::decrShareActive(s_filename); + FileServ::decrShareActive(s_filename, share_active_gen); file = NULL; } else if (pipe_file_user.get() != NULL) @@ -125,7 +125,7 @@ void ChunkSendThread::operator()(void) Server->Log("Closing file " + file->getFilename(), LL_DEBUG); Server->destroy(file); assert(!s_filename.empty()); - FileServ::decrShareActive(s_filename); + FileServ::decrShareActive(s_filename, share_active_gen); } if (cbt_hash_file_info.cbt_hash_file != NULL && cbt_hash_file_info.metadata_offset != -1) @@ -136,6 +136,7 @@ void ChunkSendThread::operator()(void) file=chunk.update_file; Server->Log("Retaining file " + file->getFilename(), LL_DEBUG); s_filename = chunk.s_filename; + share_active_gen = chunk.share_active_gen; curr_hash_size=chunk.hashsize; curr_file_size =chunk.startpos; curr_max_vdl = -1; @@ -222,7 +223,7 @@ void ChunkSendThread::operator()(void) Server->Log("Closing file (finish) " + file->getFilename(), LL_DEBUG); Server->destroy(file); assert(!s_filename.empty()); - FileServ::decrShareActive(s_filename); + FileServ::decrShareActive(s_filename, share_active_gen); file=NULL; } diff --git a/fileservplugin/ChunkSendThread.h b/fileservplugin/ChunkSendThread.h index a255849df..756ee8570 100644 --- a/fileservplugin/ChunkSendThread.h +++ b/fileservplugin/ChunkSendThread.h @@ -28,6 +28,7 @@ class ChunkSendThread : public IThread CClientThread *parent; IFile *file; std::string s_filename; + size_t share_active_gen; std::auto_ptr pipe_file_user; _i64 curr_hash_size; _i64 curr_file_size; diff --git a/fileservplugin/FileMetadataPipe.cpp b/fileservplugin/FileMetadataPipe.cpp index 0f7b8b519..59deaa0a3 100644 --- a/fileservplugin/FileMetadataPipe.cpp +++ b/fileservplugin/FileMetadataPipe.cpp @@ -160,7 +160,7 @@ bool FileMetadataPipe::readStdoutIntoBuffer( char* buf, size_t buf_avail, size_t read_bytes=0; metadata_file.reset(); - PipeSessions::fileMetadataDone(public_fn.substr(1), server_token); + PipeSessions::fileMetadataDone(public_fn.substr(1), server_token, active_gen); metadata_state = MetadataState_Wait; return false; } @@ -273,7 +273,7 @@ bool FileMetadataPipe::readStdoutIntoBuffer( char* buf, size_t buf_avail, size_t if(fn_off==sizeof(unsigned int)) { - PipeSessions::fileMetadataDone(public_fn.substr(1), server_token); + PipeSessions::fileMetadataDone(public_fn.substr(1), server_token, active_gen); metadata_state = MetadataState_Wait; } @@ -319,7 +319,7 @@ bool FileMetadataPipe::readStdoutIntoBuffer( char* buf, size_t buf_avail, size_t if(fn_off==sizeof(unsigned int)) { metadata_file.reset(); - PipeSessions::fileMetadataDone(public_fn.substr(1), server_token); + PipeSessions::fileMetadataDone(public_fn.substr(1), server_token, active_gen); metadata_state = MetadataState_Wait; } @@ -335,7 +335,7 @@ bool FileMetadataPipe::readStdoutIntoBuffer( char* buf, size_t buf_avail, size_t } if (raw_metadata.size() - metadata_buffer_off == 0) { - PipeSessions::fileMetadataDone(public_fn.substr(1), server_token); + PipeSessions::fileMetadataDone(public_fn.substr(1), server_token, active_gen); metadata_state = MetadataState_Wait; } return true; @@ -450,7 +450,7 @@ bool FileMetadataPipe::readStdoutIntoBuffer( char* buf, size_t buf_avail, size_t transmit_wait_pipe->Write(std::string()); transmit_wait_pipe = NULL; transmit_file = NULL; - PipeSessions::fileMetadataDone(public_fn.substr(1), server_token); + PipeSessions::fileMetadataDone(public_fn.substr(1), server_token, active_gen); metadata_state = MetadataState_Wait; } return true; @@ -488,13 +488,14 @@ bool FileMetadataPipe::readStdoutIntoBuffer( char* buf, size_t buf_avail, size_t msg_data.getStr(&local_fn) && msg_data.getInt64(&folder_items) && msg_data.getInt64(&metadata_id) && - msg_data.getStr(&server_token)) + msg_data.getStr(&server_token) && + msg_data.getInt64(&active_gen)) { assert(!public_fn.empty() || !local_fn.empty()); if (std::find(last_public_fns.begin(), last_public_fns.end(), public_fn) != last_public_fns.end()) { - PipeSessions::fileMetadataDone(public_fn, server_token); + PipeSessions::fileMetadataDone(public_fn, server_token, active_gen); *buf = ID_METADATA_NOP; read_bytes = 1; return true; @@ -511,10 +512,10 @@ bool FileMetadataPipe::readStdoutIntoBuffer( char* buf, size_t buf_avail, size_t if (file_type_flags == 0) { - Server->Log("Error getting file type of " + local_fn, LL_ERROR); + Server->Log("Error getting file type of " + local_fn+". "+os_last_error_str(), LL_ERROR); *buf = ID_METADATA_NOP; read_bytes = 1; - PipeSessions::fileMetadataDone(public_fn, server_token); + PipeSessions::fileMetadataDone(public_fn, server_token, active_gen); return true; } @@ -528,7 +529,7 @@ bool FileMetadataPipe::readStdoutIntoBuffer( char* buf, size_t buf_avail, size_t Server->Log("Error opening file handle to " + local_fn+". "+os_last_error_str(), LL_ERROR); *buf = ID_METADATA_NOP; read_bytes = 1; - PipeSessions::fileMetadataDone(public_fn, server_token); + PipeSessions::fileMetadataDone(public_fn, server_token, active_gen); return true; } @@ -569,7 +570,7 @@ bool FileMetadataPipe::readStdoutIntoBuffer( char* buf, size_t buf_avail, size_t *buf = ID_METADATA_NOP; read_bytes = 1; metadata_state = MetadataState_Wait; - PipeSessions::fileMetadataDone(public_fn.substr(1), server_token); + PipeSessions::fileMetadataDone(public_fn.substr(1), server_token, active_gen); return true; } @@ -585,7 +586,8 @@ bool FileMetadataPipe::readStdoutIntoBuffer( char* buf, size_t buf_avail, size_t else if (id == METADATA_PIPE_SEND_RAW && msg_data.getStr(&public_fn) && msg_data.getStr(&raw_metadata) - && msg_data.getStr(&server_token)) + && msg_data.getStr(&server_token) + && msg_data.getInt64(&active_gen)) { metadata_state = MetadataState_Raw; metadata_buffer_off = 0; @@ -595,7 +597,8 @@ bool FileMetadataPipe::readStdoutIntoBuffer( char* buf, size_t buf_avail, size_t && msg_data.getStr(&public_fn) && msg_data.getVoidPtr(reinterpret_cast(&transmit_file)) && msg_data.getVoidPtr(reinterpret_cast(&transmit_wait_pipe)) - && msg_data.getStr(&server_token)) + && msg_data.getStr(&server_token) + && msg_data.getInt64(&active_gen) ) { metadata_state = MetadataState_RawFileFnSize; *buf = ID_RAW_FILE; @@ -659,7 +662,7 @@ void FileMetadataPipe::cleanupOnForceShutdown() { if (metadata_state != MetadataState_Wait) { - PipeSessions::fileMetadataDone(public_fn.substr(1), server_token); + PipeSessions::fileMetadataDone(public_fn.substr(1), server_token, active_gen); } metadata_file.reset(); @@ -698,24 +701,27 @@ void FileMetadataPipe::cleanupOnForceShutdown() msg_data.getStr(&local_fn) && msg_data.getInt64(&folder_items) && msg_data.getInt64(&metadata_id) && - msg_data.getStr(&server_token)) + msg_data.getStr(&server_token) && + msg_data.getInt64(&active_gen) ) { - PipeSessions::fileMetadataDone(public_fn, server_token); + PipeSessions::fileMetadataDone(public_fn, server_token, active_gen); } else if (id == METADATA_PIPE_SEND_RAW && msg_data.getStr(&public_fn) && msg_data.getStr(&raw_metadata) - && msg_data.getStr(&server_token)) + && msg_data.getStr(&server_token) + && msg_data.getInt64(&active_gen)) { - PipeSessions::fileMetadataDone(public_fn.substr(1), server_token); + PipeSessions::fileMetadataDone(public_fn.substr(1), server_token, active_gen); } else if (id == METADATA_PIPE_SEND_RAW_FILEDATA && msg_data.getStr(&public_fn) && msg_data.getVoidPtr(reinterpret_cast(&transmit_file)) && msg_data.getVoidPtr(reinterpret_cast(&transmit_wait_pipe)) - && msg_data.getStr(&server_token)) + && msg_data.getStr(&server_token) + && msg_data.getInt64(&active_gen)) { - PipeSessions::fileMetadataDone(public_fn.substr(1), server_token); + PipeSessions::fileMetadataDone(public_fn.substr(1), server_token, active_gen); } } } diff --git a/fileservplugin/FileMetadataPipe.h b/fileservplugin/FileMetadataPipe.h index 8d706569f..58739c337 100644 --- a/fileservplugin/FileMetadataPipe.h +++ b/fileservplugin/FileMetadataPipe.h @@ -107,6 +107,7 @@ class FileMetadataPipe : public PipeFileBase int64 metadata_file_off; int64 metadata_file_size; int64 metadata_id; + int64 active_gen; MetadataState metadata_state; @@ -133,5 +134,10 @@ class FileMetadataPipe : public PipeFileBase #include #include "../common/data.h" +#if defined(__APPLE__) || defined(__FreeBSD__) +void serialize_stat_buf(const struct stat& buf, const std::string& symlink_target, CWData& data); +#else void serialize_stat_buf(const struct stat64& buf, const std::string& symlink_target, CWData& data); -#endif \ No newline at end of file +#endif + +#endif diff --git a/fileservplugin/FileServ.cpp b/fileservplugin/FileServ.cpp index a6de801c4..037cf73f4 100644 --- a/fileservplugin/FileServ.cpp +++ b/fileservplugin/FileServ.cpp @@ -33,7 +33,8 @@ bool FileServ::pause=false; std::map FileServ::script_mappings; IFileServ::ITokenCallbackFactory* FileServ::token_callback_factory = NULL; std::map FileServ::fn_redirects; -std::map FileServ::active_shares; +std::map, size_t> FileServ::active_shares; +size_t FileServ::active_generation = 0; FileServ::IReadErrorCallback* FileServ::read_error_callback = NULL; std::vector FileServ::read_error_files; std::map, IFileServ::CbtHashFileInfo> FileServ::cbt_hash_files; @@ -263,7 +264,7 @@ IFileServ::ITokenCallback* FileServ::newTokenCallback() return token_callback_factory->getTokenCallback(); } -void FileServ::incrShareActive(std::string sharename) +size_t FileServ::incrShareActive(std::string sharename) { if (sharename.find("/") != std::string::npos) { @@ -271,10 +272,11 @@ void FileServ::incrShareActive(std::string sharename) } IScopedLock lock(mutex); - ++active_shares[sharename]; + ++active_shares[std::make_pair(sharename, active_generation)]; + return active_generation; } -void FileServ::decrShareActive(std::string sharename) +void FileServ::decrShareActive(std::string sharename, size_t gen) { if (sharename.find("/") != std::string::npos) { @@ -283,7 +285,7 @@ void FileServ::decrShareActive(std::string sharename) IScopedLock lock(mutex); - std::map::iterator it = active_shares.find(sharename); + std::map, size_t>::iterator it = active_shares.find(std::make_pair(sharename, gen)); if (it != active_shares.end()) { @@ -304,9 +306,38 @@ bool FileServ::hasActiveTransfers(const std::string& sharename, const std::strin IScopedLock lock(mutex); - std::map::iterator it = active_shares.find(server_token + "|" + sharename); + for (std::map, size_t>::iterator it = active_shares.begin(); + it != active_shares.end();++it) + { + if (it->first.first == server_token + "|" + sharename) + { + return true; + } + } + + return false; +} + +bool FileServ::hasActiveTransfersGen(const std::string& sharename, const std::string& server_token, size_t gen) +{ + if (PipeSessions::isShareActiveGen(sharename, server_token, gen)) + { + return true; + } - return it != active_shares.end(); + IScopedLock lock(mutex); + + for (std::map, size_t>::iterator it = active_shares.begin(); + it != active_shares.end(); ++it) + { + if (it->first.first == server_token + "|" + sharename && + it->first.second<=gen) + { + return true; + } + } + + return false; } bool FileServ::registerFnRedirect(const std::string & source_fn, const std::string & target_fn) @@ -418,6 +449,13 @@ void FileServ::deregisterScriptPipeFile(const std::string & script_fn) std::map::iterator it = script_mappings.find(script_fn); if (it != script_mappings.end()) { + delete it->second.pipe_file; script_mappings.erase(it); } } + +size_t FileServ::incrActiveGeneration() +{ + IScopedLock lock(mutex); + return active_generation++; +} diff --git a/fileservplugin/FileServ.h b/fileservplugin/FileServ.h index 14ac57721..e5935b037 100644 --- a/fileservplugin/FileServ.h +++ b/fileservplugin/FileServ.h @@ -43,12 +43,14 @@ class FileServ : public IFileServ static IFileServ::ITokenCallback* newTokenCallback(); - static void incrShareActive(std::string sharename); + static size_t incrShareActive(std::string sharename); - static void decrShareActive(std::string sharename); + static void decrShareActive(std::string sharename, size_t gen); bool hasActiveTransfers(const std::string& sharename, const std::string& server_token); + bool hasActiveTransfersGen(const std::string& sharename, const std::string& server_token, size_t gen); + bool registerFnRedirect(const std::string& source_fn, const std::string& target_fn); static std::string getRedirectedFn(const std::string& source_fn); @@ -71,6 +73,8 @@ class FileServ : public IFileServ virtual void deregisterScriptPipeFile(const std::string& script_fn); + size_t incrActiveGeneration(); + private: bool *dostop; THREADPOOL_TICKET serverticket; @@ -125,7 +129,9 @@ class FileServ : public IFileServ static ITokenCallbackFactory* token_callback_factory; - static std::map active_shares; + static std::map, size_t> active_shares; + + static size_t active_generation; static IReadErrorCallback* read_error_callback; @@ -137,6 +143,7 @@ class FileServ : public IFileServ class ScopedShareActive { + size_t gen; public: ScopedShareActive() { @@ -148,7 +155,7 @@ class ScopedShareActive { if (!sharename.empty()) { - FileServ::incrShareActive(sharename); + gen = FileServ::incrShareActive(sharename); } } @@ -156,7 +163,7 @@ class ScopedShareActive { if (!sharename.empty()) { - FileServ::decrShareActive(sharename); + FileServ::decrShareActive(sharename, gen); } } @@ -164,18 +171,19 @@ class ScopedShareActive { if (!sharename.empty()) { - FileServ::decrShareActive(sharename); + FileServ::decrShareActive(sharename, gen); } sharename = new_sharename; if (!sharename.empty()) { - FileServ::incrShareActive(sharename); + gen = FileServ::incrShareActive(sharename); } } - void release() + size_t release() { sharename.clear(); + return gen; } private: diff --git a/fileservplugin/IFileServ.h b/fileservplugin/IFileServ.h index 9491e2599..feefaf4d8 100644 --- a/fileservplugin/IFileServ.h +++ b/fileservplugin/IFileServ.h @@ -57,11 +57,13 @@ class IFileServ : public IObject virtual void removeMetadataCallback(const std::string &name, const std::string& identity) = 0; virtual void registerTokenCallbackFactory(ITokenCallbackFactory* callback_factory) = 0; virtual bool hasActiveTransfers(const std::string& sharename, const std::string& server_token) = 0; + virtual bool hasActiveTransfersGen(const std::string& sharename, const std::string& server_token, size_t gen) = 0; virtual bool registerFnRedirect(const std::string& source_fn, const std::string& target_fn) = 0; virtual void registerReadErrorCallback(IReadErrorCallback* cb) = 0; virtual void registerScriptPipeFile(const std::string& script_fn, IPipeFileExt* pipe_file) = 0; virtual void deregisterScriptPipeFile(const std::string& script_fn) = 0; virtual void clearReadErrors() = 0; + virtual size_t incrActiveGeneration() = 0; struct CbtHashFileInfo { diff --git a/fileservplugin/PipeFileTar.h b/fileservplugin/PipeFileTar.h index 38606dcb7..78dc5186a 100644 --- a/fileservplugin/PipeFileTar.h +++ b/fileservplugin/PipeFileTar.h @@ -156,3 +156,7 @@ class PipeFileTar : public IPipeFile std::string identity; }; + +#if defined(_WIN32) || defined(__APPLE__) || defined(__FreeBSD__) +#undef stat64 +#endif diff --git a/fileservplugin/PipeSessions.cpp b/fileservplugin/PipeSessions.cpp index 2d02ac254..34bae72d4 100644 --- a/fileservplugin/PipeSessions.cpp +++ b/fileservplugin/PipeSessions.cpp @@ -32,10 +32,12 @@ volatile bool PipeSessions::do_stop = false; IMutex* PipeSessions::mutex = NULL; +IMutex* PipeSessions::active_shares_mutex = NULL; std::map PipeSessions::pipe_files; std::map PipeSessions::exit_information; std::map, IFileServ::IMetadataCallback*> PipeSessions::metadata_callbacks; -std::map PipeSessions::active_shares; +std::map, size_t> PipeSessions::active_shares; +size_t PipeSessions::active_shares_gen = 0; const int64 pipe_file_timeout = 1*60*60*1000; const int64 pipe_file_read_timeout = 30 * 60 * 1000; @@ -173,6 +175,7 @@ void PipeSessions::injectPipeSession(const std::string & session_key, int backup void PipeSessions::init() { mutex = Server->createMutex(); + active_shares_mutex = Server->createMutex(); Server->getThreadPool()->execute(new PipeSessions, "PipeSession: timeout"); } @@ -180,6 +183,7 @@ void PipeSessions::init() void PipeSessions::destroy() { delete mutex; + delete active_shares_mutex; do_stop=true; } @@ -406,21 +410,26 @@ IFileServ::IMetadataCallback* PipeSessions::transmitFileMetadata( const std::str IScopedLock lock(mutex); - std::map, IFileServ::IMetadataCallback*>::iterator iter_cb = - metadata_callbacks.find(std::make_pair(sharename, identity)); - IFileServ::IMetadataCallback* ret = NULL; - if(iter_cb!=metadata_callbacks.end()) - { - data.addVoidPtr(iter_cb->second); - ret = iter_cb->second; - } std::map::iterator it = pipe_files.find("urbackup/FILE_METADATA|"+server_token); if(it!=pipe_files.end() && it->second.input_pipe!=NULL) { - ++active_shares[sharename + "|" + server_token]; + { + IScopedLock alock(active_shares_mutex); + ++active_shares[std::make_pair(sharename + "|" + server_token, active_shares_gen)]; + data.addUInt64(active_shares_gen); + } + + std::map, IFileServ::IMetadataCallback*>::iterator iter_cb = + metadata_callbacks.find(std::make_pair(sharename, identity)); + + if(iter_cb!=metadata_callbacks.end()) + { + data.addVoidPtr(iter_cb->second); + ret = iter_cb->second; + } it->second.input_pipe->Write(data.getDataPtr(), data.getDataSize()); } @@ -457,7 +466,11 @@ void PipeSessions::transmitFileMetadata(const std::string & public_fn, const std if (it != pipe_files.end() && it->second.input_pipe != NULL) { - ++active_shares[sharename + "|" + server_token]; + { + IScopedLock alock(active_shares_mutex); + ++active_shares[std::make_pair(sharename + "|" + server_token, active_shares_gen)]; + data.addUInt64(active_shares_gen); + } it->second.input_pipe->Write(data.getDataPtr(), data.getDataSize()); } @@ -500,12 +513,15 @@ void PipeSessions::transmitFileMetadataAndFiledataWait(const std::string & publi if (it != pipe_files.end() && it->second.input_pipe != NULL) { - ++active_shares[sharename + "|" + server_token]; + { + IScopedLock alock(active_shares_mutex); + active_shares[std::make_pair(sharename + "|" + server_token, active_shares_gen)]+=2; + metadatamsg.addUInt(active_shares_gen); + datamsg.addUInt(active_shares_gen); + } it->second.input_pipe->Write(datamsg.getDataPtr(), datamsg.getDataSize()); - ++active_shares[sharename + "|" + server_token]; - it->second.input_pipe->Write(metadatamsg.getDataPtr(), metadatamsg.getDataSize()); lock.relock(NULL); @@ -520,7 +536,7 @@ void PipeSessions::transmitFileMetadataAndFiledataWait(const std::string & publi -void PipeSessions::fileMetadataDone(const std::string & public_fn, const std::string& server_token) +void PipeSessions::fileMetadataDone(const std::string & public_fn, const std::string& server_token, size_t active_gen) { std::string sharename = getuntil("/", public_fn); if (sharename.empty()) @@ -528,9 +544,9 @@ void PipeSessions::fileMetadataDone(const std::string & public_fn, const std::st sharename = public_fn; } - IScopedLock lock(mutex); + IScopedLock lock(active_shares_mutex); - std::map::iterator it = active_shares.find(sharename + "|" + server_token); + std::map, size_t>::iterator it = active_shares.find(std::make_pair(sharename + "|" + server_token, active_gen)); if (it != active_shares.end()) { @@ -544,11 +560,37 @@ void PipeSessions::fileMetadataDone(const std::string & public_fn, const std::st bool PipeSessions::isShareActive(const std::string & sharename, const std::string& server_token) { - IScopedLock lock(mutex); + IScopedLock lock(active_shares_mutex); + + for (std::map, size_t>::iterator it = active_shares.begin(); + it != active_shares.end(); ++it) + { + if (it->first.first == sharename + "|" + server_token) + return true; + } - std::map::iterator it = active_shares.find(sharename + "|" + server_token); + return false; +} + +bool PipeSessions::isShareActiveGen(const std::string& sharename, const std::string& server_token, size_t gen) +{ + IScopedLock lock(active_shares_mutex); - return it != active_shares.end(); + for (std::map, size_t>::iterator it = active_shares.begin(); + it != active_shares.end(); ++it) + { + if (it->first.first == sharename + "|" + server_token && + it->first.second<=gen) + return true; + } + + return false; +} + +void PipeSessions::setActiveSharesGen(size_t gen) +{ + IScopedLock lock(active_shares_mutex); + active_shares_gen = gen; } void PipeSessions::metadataStreamEnd( const std::string& server_token ) diff --git a/fileservplugin/PipeSessions.h b/fileservplugin/PipeSessions.h index 1157309c4..3933a6ca0 100644 --- a/fileservplugin/PipeSessions.h +++ b/fileservplugin/PipeSessions.h @@ -73,10 +73,14 @@ class PipeSessions : public IThread static void transmitFileMetadataAndFiledataWait(const std::string& public_fn, const std::string& metadata, const std::string& server_token, const std::string& identity, IFile* file); - static void fileMetadataDone(const std::string& public_fn, const std::string& server_token); + static void fileMetadataDone(const std::string& public_fn, const std::string& server_token, size_t active_gen); static bool isShareActive(const std::string& sharename, const std::string& server_token); + static bool isShareActiveGen(const std::string& sharename, const std::string& server_token, size_t gen); + + static void setActiveSharesGen(size_t gen); + static void metadataStreamEnd(const std::string& server_token); static void phashEnd(const std::string& server_token, const std::string& phash_fn); @@ -91,9 +95,11 @@ class PipeSessions : public IThread static std::string getKey(const std::string& cmd, int& backupnum, int64& fn_random); static IMutex* mutex; + static IMutex* active_shares_mutex; static volatile bool do_stop; static std::map pipe_files; static std::map exit_information; static std::map, IFileServ::IMetadataCallback*> metadata_callbacks; - static std::map active_shares; + static std::map, size_t> active_shares; + static size_t active_shares_gen; }; \ No newline at end of file diff --git a/fileservplugin/fileservplugin.vcxproj b/fileservplugin/fileservplugin.vcxproj index fea7332ca..df468fd31 100644 --- a/fileservplugin/fileservplugin.vcxproj +++ b/fileservplugin/fileservplugin.vcxproj @@ -1,6 +1,10 @@  + + Debug + ARM64 + Debug Win32 @@ -9,6 +13,10 @@ Debug x64 + + Release + ARM64 + Release Win32 @@ -29,23 +37,34 @@ DynamicLibrary Unicode true - v142 + v143 DynamicLibrary Unicode - v142 + v143 DynamicLibrary Unicode true - v142 + v143 + + + DynamicLibrary + Unicode + true + v143 DynamicLibrary Unicode - v142 + v143 + + + DynamicLibrary + Unicode + v143 @@ -59,9 +78,15 @@ + + + + + + <_ProjectFileVersion>10.0.30319.1 @@ -72,23 +97,50 @@ $(Configuration)\ false $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ $(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ true + true $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ $(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ false + false AllRules.ruleset AllRules.ruleset + AllRules.ruleset + + AllRules.ruleset AllRules.ruleset + AllRules.ruleset + + + + + true + + + x64-windows-static-md + + + x64-windows-static-md + + + x86-windows-static-md + + + arm64-windows-static-md @@ -123,13 +175,14 @@ $(CryptoppIncludeDir);$(SolutionDir)/deps/include/cryptopp; - cryptlib_x86.lib;ws2_32.lib;%(AdditionalDependencies) + ws2_32.lib;%(AdditionalDependencies) true Console true true MachineX86 - $(CryptoppLibDir);$(SolutionDir)/deps/libs; + + @@ -154,6 +207,25 @@ MachineX64 + + + + Disabled + WIN32;_DEBUG;_CONSOLE;DO_NOT_USE_CRYPTOPP_MD5;DO_NOT_USE_CRYPTOPP_SHA;%(PreprocessorDefinitions) + true + EnableFastChecks + MultiThreadedDebugDLL + + + Level3 + ProgramDatabase + + + ws2_32.lib;%(AdditionalDependencies) + true + Console + + X64 @@ -171,13 +243,38 @@ $(CryptoppIncludeDir);$(SolutionDir)/deps/include/cryptopp; - cryptlib_x86_64.lib;ws2_32.lib;%(AdditionalDependencies) + ws2_32.lib;%(AdditionalDependencies) true Console true true MachineX64 - $(CryptoppLibDir);$(SolutionDir)/deps/libs; + + + + + + + + MaxSpeed + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + MultiThreadedDLL + true + + + Level3 + ProgramDatabase + $(CryptoppIncludeDir);$(SolutionDir)/deps/include/cryptopp; + + + ws2_32.lib;%(AdditionalDependencies) + true + Console + true + true + + diff --git a/fsimageplugin/CompressedFile.cpp b/fsimageplugin/CompressedFile.cpp index e26e03ec4..0291fa7ce 100644 --- a/fsimageplugin/CompressedFile.cpp +++ b/fsimageplugin/CompressedFile.cpp @@ -82,7 +82,7 @@ CompressedFile::CompressedFile( std::string pFilename, int pMode, size_t n_threa CompressedFile::CompressedFile(IFile* file, bool openExisting, bool readOnly, size_t n_threads) : hotCache(NULL), error(false), currentPosition(0), finished(false), uncompressedFile(file), filesize(0), readOnly(readOnly), - noMagic(false), n_threads(n_threads), numBlockOffsets(0) + noMagic(false), mutex(Server->createMutex()), n_threads(n_threads), numBlockOffsets(0) { if(openExisting) { @@ -379,7 +379,6 @@ bool CompressedFile::fillCache( __int64 offset, bool errorMsg, bool *has_error) #ifndef NO_ZSTD_COMPRESSION else if (mode == mode_zstd) { - rdecomp = blocksize; const size_t rc = ZSTD_decompress(buf, blocksize, compressedBuffer.data(), compressedSize); @@ -388,6 +387,7 @@ bool CompressedFile::fillCache( __int64 offset, bool errorMsg, bool *has_error) Server->Log(std::string("Error while decompressing file (zstd). Error code ") + ZSTD_getErrorName(rc), LL_ERROR); return false; } + rdecomp = rc; } #endif else @@ -528,6 +528,8 @@ void CompressedFile::evictFromLruCache( const SCacheItem& item ) { error=true; Server->Log("Error while writing compressed data to file", LL_ERROR); + IScopedLock lock(mutex.get()); + returnCompressedBuffer(compBuffer, compBufferIdx); return; } @@ -634,7 +636,8 @@ void CompressedFile::returnCompressedBuffer(char* buf, size_t compressed_buffer_ bool CompressedFile::finish() { - assert(!finished); + // TODO: Fix + // assert(!finished); if(hotCache.get()) { @@ -715,6 +718,58 @@ _u32 CompressedFile::writeToFile(int64 offset, const char* buffer, _u32 bsize) return written; } +void CompressedFile::resetSparseExtentIter() +{ +} + +IFsFile::SSparseExtent CompressedFile::nextSparseExtent() +{ + return SSparseExtent(); +} + +bool CompressedFile::Resize(int64 new_size, bool set_sparse) +{ + IScopedLock lock(mutex.get()); + if (new_size > filesize) + { + const size_t blockIdx = static_cast(new_size / blocksize); + const size_t currNumBlockOffsets = blockOffsets.size(); + if (blockOffsets.size() <= blockIdx) + { + const size_t new_size = (blockIdx + 1) * 2; + blockOffsets.resize(new_size); + for (size_t i = currNumBlockOffsets; i < new_size; ++i) + { + blockOffsets[i] = -1; + } + } + filesize = new_size; + numBlockOffsets = (std::max)(numBlockOffsets, blockIdx + 1); + } + return true; +} + +std::vector CompressedFile::getFileExtents(int64 starting_offset, int64 block_size, bool& more_data) +{ + more_data = false; + return std::vector(); +} + +IVdlVolCache* CompressedFile::createVdlVolCache() +{ + return NULL; +} + +int64 CompressedFile::getValidDataLength(IVdlVolCache* vol_cache) +{ + return int64(); +} + +IFsFile::os_file_handle CompressedFile::getOsHandle(bool release_handle) +{ + return os_file_handle(); +} + bool CompressedFile::hasNoMagic() { return noMagic; diff --git a/fsimageplugin/CompressedFile.h b/fsimageplugin/CompressedFile.h index bafcaf349..bc52f13b5 100644 --- a/fsimageplugin/CompressedFile.h +++ b/fsimageplugin/CompressedFile.h @@ -27,7 +27,7 @@ class ICacheEvictionCallback friend class LRUMemCache; }; -class CompressedFile : public IFile, public ICacheEvictionCallback +class CompressedFile : public IFsFile, public ICacheEvictionCallback { public: CompressedFile(std::string pFilename, int pMode, size_t n_threads); @@ -57,6 +57,14 @@ class CompressedFile : public IFile, public ICacheEvictionCallback bool hasNoMagic(); + void resetSparseExtentIter(); + SSparseExtent nextSparseExtent(); + bool Resize(int64 new_size, bool set_sparse); + std::vector getFileExtents(int64 starting_offset, int64 block_size, bool& more_data); + IVdlVolCache* createVdlVolCache(); + int64 getValidDataLength(IVdlVolCache* vol_cache); + os_file_handle getOsHandle(bool release_handle); + private: void readHeader(bool *has_error); void readIndex(bool *has_error); diff --git a/fsimageplugin/FSImageFactory.cpp b/fsimageplugin/FSImageFactory.cpp index 3f4e7bba7..0e09fe13b 100644 --- a/fsimageplugin/FSImageFactory.cpp +++ b/fsimageplugin/FSImageFactory.cpp @@ -1,6 +1,6 @@ /************************************************************************* * UrBackup - Client/Server backup system -* Copyright (C) 2011-2016 Martin Raiber +* Copyright (C) 2011-2021 Martin Raiber * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -28,6 +28,7 @@ #endif #include "fs/unknown.h" #include "vhdfile.h" +#include "vhdxfile.h" #include "../stringtools.h" #ifdef _WIN32 #include @@ -545,6 +546,9 @@ IVHDFile *FSImageFactory::createVHDFile(const std::string &fn, bool pRead_only, case ImageFormat_VHD: case ImageFormat_CompressedVHD: return new VHDFile(fn, pRead_only, pDstsize, pBlocksize, fast_mode, format!=ImageFormat_VHD); + case ImageFormat_VHDX: + case ImageFormat_CompressedVHDX: + return new VHDXFile(fn, pRead_only, pDstsize, pBlocksize, fast_mode, format != ImageFormat_VHDX); case ImageFormat_RawCowFile: #if !defined(__APPLE__) return new CowFile(fn, pRead_only, pDstsize); @@ -563,6 +567,9 @@ IVHDFile *FSImageFactory::createVHDFile(const std::string &fn, const std::string case ImageFormat_VHD: case ImageFormat_CompressedVHD: return new VHDFile(fn, parent_fn, pRead_only, fast_mode, format!=ImageFormat_VHD, pDstsize); + case ImageFormat_VHDX: + case ImageFormat_CompressedVHDX: + return new VHDXFile(fn, parent_fn, pRead_only, fast_mode, format != ImageFormat_VHDX, pDstsize); case ImageFormat_RawCowFile: #if !defined(__APPLE__) return new CowFile(fn, parent_fn, pRead_only, pDstsize); diff --git a/fsimageplugin/IFSImageFactory.h b/fsimageplugin/IFSImageFactory.h index c5ff88b6a..bc6533f0c 100644 --- a/fsimageplugin/IFSImageFactory.h +++ b/fsimageplugin/IFSImageFactory.h @@ -35,7 +35,9 @@ class IFSImageFactory : public IPlugin { ImageFormat_VHD=0, ImageFormat_CompressedVHD=1, - ImageFormat_RawCowFile=2 + ImageFormat_RawCowFile=2, + ImageFormat_VHDX = 3, + ImageFormat_CompressedVHDX = 4 }; virtual IVHDFile *createVHDFile(const std::string &fn, bool pRead_only, uint64 pDstsize, diff --git a/fsimageplugin/ImdiskSrv.cpp b/fsimageplugin/ImdiskSrv.cpp index 8e7b85bb7..86724f818 100644 --- a/fsimageplugin/ImdiskSrv.cpp +++ b/fsimageplugin/ImdiskSrv.cpp @@ -6,8 +6,8 @@ #include "FileWrapper.h" #include #include -#include -#include +#include "../external/imdisk/imdproxy.h" +#include "../external/imdisk/imdisk.h" #include #include #include diff --git a/fsimageplugin/LRUMemCache.cpp b/fsimageplugin/LRUMemCache.cpp index 47c868cd1..5bb30fb6d 100644 --- a/fsimageplugin/LRUMemCache.cpp +++ b/fsimageplugin/LRUMemCache.cpp @@ -77,6 +77,8 @@ bool LRUMemCache::put( __int64 offset, const char* buffer, size_t bsize ) SCacheItem newItem = createInt(offset); + memset(newItem.buffer, 0, buffersize); + size_t innerOffset = static_cast(offset-newItem.offset); if( buffersize - innerOffset < bsize) diff --git a/fsimageplugin/cowfile.cpp b/fsimageplugin/cowfile.cpp index 135a9563c..1c9ad3016 100644 --- a/fsimageplugin/cowfile.cpp +++ b/fsimageplugin/cowfile.cpp @@ -529,6 +529,7 @@ void CowFile::setupBitmap() uint64 n_blocks = filesize/blocksize+(filesize%blocksize>0?1:0); size_t n_bits = n_blocks/8 + (n_blocks%8>0?1:0); bitmap.resize(n_bits); + bitmap_dirty=true; } void CowFile::resizeBitmap() @@ -828,7 +829,8 @@ bool CowFile::trimUnused(_i64 fs_offset, _i64 trim_blocksize, ITrimCallback* tri unused_end = filesize; } - if(hasBitmapRangeNarrow(unused_start, unused_end, trim_blocksize_bytes)) + if(hasBitmapRangeNarrow(unused_start, unused_end, trim_blocksize_bytes) + && unused_end>unused_start) { if(!setUnused(unused_start, unused_end)) { @@ -849,7 +851,8 @@ bool CowFile::trimUnused(_i64 fs_offset, _i64 trim_blocksize, ITrimCallback* tri int64 unused_start = fs_offset + unused_start_block*bitmap_blocksize; int64 unused_end = filesize; - if(hasBitmapRangeNarrow(unused_start, unused_end, trim_blocksize_bytes)) + if(hasBitmapRangeNarrow(unused_start, unused_end, trim_blocksize_bytes) + && unused_end>unused_start) { if(!setUnused(unused_start, unused_end)) { diff --git a/fsimageplugin/dllmain.cpp b/fsimageplugin/dllmain.cpp index 065ed4866..82b192477 100644 --- a/fsimageplugin/dllmain.cpp +++ b/fsimageplugin/dllmain.cpp @@ -1,6 +1,6 @@ /************************************************************************* * UrBackup - Client/Server backup system -* Copyright (C) 2011-2016 Martin Raiber +* Copyright (C) 2011-2021 Martin Raiber * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -54,6 +54,7 @@ extern IServer* Server; #include #include "vhdfile.h" +#include "vhdxfile.h" #ifndef _WIN32 #include "cowfile.h" #endif @@ -70,6 +71,7 @@ extern IServer* Server; #include "win_dialog.h" #endif #include "FileWrapper.h" +#include "FSImageFactory.h" #ifdef __linux__ #include @@ -129,6 +131,28 @@ namespace } } + if (findextension(fn) == "vhdxz") + { + std::auto_ptr vhdfile(new VHDXFile(fn, true, 0)); + + if (vhdfile->isOpen() && vhdfile->getParent() != NULL) + { + if (vhdfile->isCompressed()) + { + std::string parent_fn = vhdfile->getParent()->getFilename(); + vhdfile.reset(); + Server->Log("Decompressing parent VHDX \"" + parent_fn + "\"...", LL_INFO); + bool b = decompress_vhd(parent_fn, parent_fn); + + if (!b) + { + Server->Log("Error decompressing parent VHDX", LL_ERROR); + return false; + } + } + } + } + { CompressedFile compFile(fn, MODE_READ, 0); @@ -189,17 +213,28 @@ namespace } - IVHDFile* open_device_file(std::string device_verify) + IVHDFile* open_device_file(std::string device_verify, bool read_only=true, int64 dst_size = 0, + std::string parent_fn=std::string(), bool fast_mode=false) { std::string ext = strlower(findextension(device_verify)); if(ext=="vhd" || ext=="vhdz") { - return new VHDFile(device_verify, true,0); + if(parent_fn.empty()) + return new VHDFile(device_verify, read_only, dst_size, 2*1024*1024, fast_mode); + else + return new VHDFile(device_verify, parent_fn, read_only, fast_mode); + } + else if (ext == "vhdx" || ext == "vhdxz") + { + if (parent_fn.empty()) + return new VHDXFile(device_verify, read_only, dst_size, 2 * 1024 * 1024, fast_mode); + else + return new VHDXFile(device_verify, parent_fn, read_only, fast_mode); } #if !defined(_WIN32) && !defined(__APPLE__) else if(ext=="raw") { - return new CowFile(device_verify, true,0); + return new CowFile(device_verify, read_only, dst_size); } #endif else @@ -242,7 +277,7 @@ namespace if(fn.empty()) { - Server->Log("No input VHD", LL_ERROR); + Server->Log("No input VHD(X)", LL_ERROR); return false; } @@ -283,9 +318,13 @@ namespace mbrdatas.push_back(mbrdata); } - std::string mbr = mbrdatas[0].mbr_data; + if (mbrdatas.empty()) + { + Server->Log("Could not open any MBR data", LL_ERROR); + return false; + } - partition* partitions = reinterpret_cast(&mbr[446]); + SMBRData& mbrdata = mbrdatas[0]; std::string skip_s=Server->getServerParameter("skip"); int skip=1024*512; @@ -293,7 +332,6 @@ namespace { skip=atoi(skip_s.c_str()); } - std::vector input_files; std::vector input_fs; @@ -306,6 +344,12 @@ namespace int64 total_size = 0; int64 total_written = 0; + FSImageFactory img_fak; + + bool gpt_style = false; + std::vector partitions = img_fak.readPartitions(mbrdata.mbr_data, + mbrdata.gpt_header, mbrdata.gpt_table, gpt_style); + for(size_t i=0;iSize(); } - partition* cpart = partitions + (mbrdatas[i].partition_number-1); - - /*unsigned char chs_expected[3] = { 0xfe, 0xff, 0xff }; - if(memcmp(cpart->chs_begin, chs_expected, 3)!=0 - || memcmp(cpart->chs_end, chs_expected,3)!=0) + if (mbrdatas[i].partition_number >= partitions.size()) { - Server->Log(L"MBR partition of Volume "+mbrdatas[i].volume_name+L" does not use LBA addressing scheme", LL_ERROR); + Server->Log("Partitions number of input file " + fn[i] + + " larger than parsed partitions from MBR/GPT", LL_ERROR); return false; - }*/ + } - cpart->start_sector=little_endian(cpart->start_sector); - cpart->nr_sector=little_endian(cpart->nr_sector); + IFSImageFactory::SPartition partition = partitions[mbrdatas[i].partition_number]; + + total_size = (std::max)(total_size, partition.offset + partition.length); + } - total_size = (std::max)(total_size, cpart->start_sector*c_sector_size + cpart->nr_sector*c_sector_size); + if (gpt_style) + { + total_size = (std::max)(total_size, mbrdata.backup_gpt_header_pos + 512); } - VHDFile vhdout(output, false, total_size, 2*1024*1024, true, false); - if(!vhdout.isOpen()) + int64 curr_pos=0; + + std::auto_ptr vhdout(open_device_file(output, false, total_size, std::string(), true)); + IFile* vhdout_file = reinterpret_cast(vhdout.get()); + + if (vhdout.get()==NULL || !vhdout->isOpen()) { - Server->Log("Error opening output VHD-File \""+output+"\"", LL_ERROR); + Server->Log("Error opening output VHD(X)-File \"" + output + "\"", LL_ERROR); return false; } - int64 curr_pos=0; - - vhdout.Seek(0); + vhdout->Seek(0); - if(vhdout.Write(mbr.data(), static_cast<_u32>(mbr.size()))!=static_cast<_u32>(mbr.size())) + if(vhdout->Write(mbrdata.mbr_data.data(), static_cast<_u32>(mbrdata.mbr_data.size())) + !=static_cast<_u32>(mbrdata.mbr_data.size())) { Server->Log("Error writing MBR", LL_ERROR); return false; } + if (mbrdata.gpt_style) + { + Server->Log("Writing GPT header..."); + if (vhdout_file->Write(mbrdata.gpt_header_pos, mbrdata.gpt_header) + != mbrdata.gpt_header.size()) + { + Server->Log("Writing GPT header failed. " + os_last_error_str(), LL_ERROR); + } + + Server->Log("Writing GPT table..."); + if (vhdout_file->Write(mbrdata.gpt_table_pos, mbrdata.gpt_table) + != mbrdata.gpt_table.size()) + { + Server->Log("Writing GPT table failed. " + os_last_error_str(), LL_ERROR); + } + + if (mbrdata.backup_gpt_header_pos != -1) + { + Server->Log("Writing GPT backup header..."); + if (vhdout_file->Write(mbrdata.backup_gpt_header_pos, mbrdata.backup_gpt_header) + != mbrdata.backup_gpt_header.size()) + { + Server->Log("Writing GPT backup header failed. " + os_last_error_str(), LL_ERROR); + } + } + + if (mbrdata.backup_gpt_table_pos != -1) + { + Server->Log("Writing GPT backup table..."); + if (vhdout_file->Write(mbrdata.backup_gpt_table_pos, mbrdata.backup_gpt_table) + != mbrdata.backup_gpt_table.size()) + { + Server->Log("Writing backup GPT table failed. " + os_last_error_str(), LL_ERROR); + } + } + } + + if (mbrdata.extra_data_pos != -1) + { + Server->Log("Writing extra data at position " + convert(mbrdata.extra_data_pos) + + " size " + convert(mbrdata.extra_data.size()) + " ..."); + if (vhdout_file->Write(mbrdata.extra_data_pos, mbrdata.extra_data) != mbrdata.extra_data.size()) + { + Server->Log("Writing extra data failed. " + os_last_error_str(), LL_ERROR); + } + } + for(size_t i=0;iLog("Writing "+fn[i]+" into output VHD..."); + IFSImageFactory::SPartition cpart = partitions[mbrdatas[i].partition_number]; - partition* cpart = partitions + (mbrdatas[i].partition_number-1); - int64 out_pos = cpart->start_sector*c_sector_size; - int64 max_pos = out_pos + cpart->nr_sector*c_sector_size; + int64 out_pos = cpart.offset; + int64 max_pos = cpart.offset + cpart.length; + + Server->Log("Writing "+fn[i]+" into output VHD(x)... Partition=" + + convert(mbrdatas[i].partition_number)+" offset=" + convert(out_pos) + +" length="+PrettyPrintBytes(cpart.length), LL_ERROR); if(input_fs[i]!=NULL) { @@ -386,8 +484,8 @@ namespace { fs_buffer fsb(input_fs[i], buf); - vhdout.Seek(out_pos); - if(vhdout.Write(buf->getBuf(), static_cast<_u32>(blocksize))!=static_cast<_u32>(blocksize)) + vhdout->Seek(out_pos); + if(vhdout->Write(buf->getBuf(), static_cast<_u32>(blocksize))!=static_cast<_u32>(blocksize)) { Server->Log("Error writing to VHD output file", LL_ERROR); return false; @@ -412,7 +510,7 @@ namespace std::vector buf; buf.resize(32768); - vhdout.Seek(out_pos); + vhdout->Seek(out_pos); if(out_pos+input_files[i]->Size()>max_pos) { @@ -434,7 +532,7 @@ namespace return false; } - if(vhdout.Write(buf.data(), read)!=read) + if(vhdout->Write(buf.data(), read)!=read) { Server->Log("Error writing to VHD output file (2)", LL_ERROR); return false; @@ -621,9 +719,9 @@ DLLEXPORT void LoadActions(IServer* pServer) if(decompress=="SelectViaGUI") { std::string filter; - filter += "Compressed image files (*.vhdz)"; + filter += "Compressed image files (*.vhdz;*.vhdxz)"; filter += '\0'; - filter += "*.vhdz"; + filter += "*.vhdz;*.vhdxz"; filter += '\0'; filter += '\0'; std::vector res = file_via_dialog("Please select compressed image file to decompress", @@ -650,7 +748,8 @@ DLLEXPORT void LoadActions(IServer* pServer) #endif std::string targetName = decompress; - if(findextension(decompress)!="vhdz" && findextension(decompress)!="urz") + if(findextension(decompress)!="vhdz" && findextension(decompress)!="urz" + && findextension(decompress) != "vhdxz") { Server->Log("Unknown file extension: "+findextension(decompress), LL_ERROR); exit(1); @@ -676,9 +775,9 @@ DLLEXPORT void LoadActions(IServer* pServer) if(assemble=="SelectViaGUI") { std::string filter; - filter += "Image files (*.vhdz;*.vhd)"; + filter += "Image files (*.vhdz;*.vhd;*.vhdxz;*.vhdx)"; filter += '\0'; - filter += "*.vhdz;*.vhd"; + filter += "*.vhdz;*.vhd;*.vhdxz;*.vhdx"; filter += '\0'; /*filter += L"Image files (*.vhd)"; filter += '\0'; @@ -698,10 +797,14 @@ DLLEXPORT void LoadActions(IServer* pServer) filter += '\0'; filter += "*.vhd"; filter += '\0'; + filter += "Image file v2 (*.vhdx)"; + filter += '\0'; + filter += "*.vhdx"; + filter += '\0'; filter += '\0'; std::vector output_files = file_via_dialog("Please select where to save the output image", - filter, false, false, "vhd"); + filter, false, false, "vhdx"); if(!output_files.empty()) { @@ -773,7 +876,7 @@ DLLEXPORT void LoadActions(IServer* pServer) else { IFile *out=Server->openFile(vhdcopy_out, MODE_RW); - if(out==NULL) + if(out==NULL && vhdcopy_out!="null") { Server->Log("Couldn't open output file", LL_ERROR); exit(6); @@ -781,10 +884,10 @@ DLLEXPORT void LoadActions(IServer* pServer) else { std::string skip_s=Server->getServerParameter("skip"); - int skip=1024*512; + int64 skip=1024*512; if(!skip_s.empty()) { - skip=atoi(skip_s.c_str()); + skip=watoi64(skip_s); } else if (is_disk_mbr(vhdcopy_in + ".mbr")) { @@ -800,11 +903,13 @@ DLLEXPORT void LoadActions(IServer* pServer) uint64 currpos=skip; bool is_ok=true; - out->Seek(0); + if(out) + out->Seek(0); + while(currpos%vhd_blocksize!=0) { is_ok=in->Read(buffer, 512, read); - if(read>0) + if(read>0 && out) { _u32 rc=out->Write(buffer, (_u32)read); if(rc!=read) @@ -826,7 +931,7 @@ DLLEXPORT void LoadActions(IServer* pServer) if(in->has_sector()) { is_ok=in->Read(buffer, 4096, read); - if(read>0) + if(read>0 && out) { _u32 rc=out->Write(buffer, (_u32)read); if(rc!=read) @@ -835,6 +940,10 @@ DLLEXPORT void LoadActions(IServer* pServer) exit(7); } } + if (!is_ok && currpos + read < in->getSize()) + { + Server->Log("Error reading from input file. " + os_last_error_str(), LL_ERROR); + } currpos+=read; } else @@ -842,7 +951,8 @@ DLLEXPORT void LoadActions(IServer* pServer) read=4096; currpos+=read; in->Seek(currpos); - out->Seek(currpos-skip); + if(out) + out->Seek(currpos-skip); } ++p_skip; @@ -865,6 +975,135 @@ DLLEXPORT void LoadActions(IServer* pServer) } } } + + std::string vhdmake_in = Server->getServerParameter("vhdmake_in"); + if (!vhdmake_in.empty()) + { + Server->Log("VHDMake."); + + std::auto_ptr inFile(Server->openFile(vhdmake_in, MODE_READ)); + + if (inFile.get() == NULL) + { + Server->Log("Error opening " + vhdmake_in + ". ", LL_ERROR); + exit(1); + } + + std::string vhd_out = Server->getServerParameter("vhd_out"); + IVHDFile* vhdfile = open_device_file(vhd_out, false, inFile->Size()); + if (vhdfile == NULL) + { + Server->Log("Error opening " + vhd_out + ". ", LL_ERROR); + exit(1); + } + + std::vector buf(512 * 1024); + + for (int64 pos = 0, size = inFile->Size(); pos < size; pos += buf.size()) + { + _u32 towrite = static_cast<_u32>((std::min)(static_cast(buf.size()), size - pos)); + if (inFile->Read(pos, buf.data(), towrite) != towrite) + { + Server->Log("Error reading from in file", LL_ERROR); + exit(2); + } + + if (vhdfile->Write(buf.data(), towrite) != towrite) + { + Server->Log("Error writing to vhd file", LL_ERROR); + exit(2); + } + } + + delete vhdfile; + Server->Log("VHDMake complete", LL_INFO); + exit(0); + } + + std::string vhdmake_diff_in = Server->getServerParameter("vhdmake_diff_in"); + if (!vhdmake_diff_in.empty()) + { + Server->Log("VHDMake Diff."); + + std::auto_ptr inFile(Server->openFile(vhdmake_diff_in, MODE_READ)); + + if (inFile.get() == NULL) + { + Server->Log("Error opening " + vhdmake_diff_in + ". ", LL_ERROR); + exit(1); + } + + std::string vhd_out_parent = Server->getServerParameter("vhd_out_parent"); + + if (vhd_out_parent.empty() || !FileExists(vhd_out_parent)) + { + Server->Log("Error finding vhd_out_parent \"" + vhd_out_parent + "\"", LL_ERROR); + exit(2); + } + + std::string vhdmake_in_parent = Server->getServerParameter("vhdmake_in_parent"); + + if (vhdmake_in_parent.empty() || !FileExists(vhdmake_in_parent)) + { + Server->Log("Error finding vhdmake_in_parent \"" + vhdmake_in_parent + "\"", LL_ERROR); + exit(2); + } + + std::auto_ptr inParentFile(Server->openFile(vhdmake_in_parent, MODE_READ)); + + if (inParentFile.get() == NULL) + { + Server->Log("Error opening " + vhdmake_in_parent + ". ", LL_ERROR); + exit(1); + } + + std::string vhd_out = Server->getServerParameter("vhd_out"); + Server->deleteFile(vhd_out); //TODO: rm + IVHDFile* vhdfile = open_device_file(vhd_out, false, inFile->Size(), vhd_out_parent); + if (vhdfile == NULL) + { + Server->Log("Error opening " + vhd_out + ". ", LL_ERROR); + exit(1); + } + + std::vector buf(512); + std::vector buf_prev(512); + + int64 written = 0; + for (int64 pos = 0, size = inFile->Size(); pos < size; pos += buf.size()) + { + _u32 towrite = static_cast<_u32>((std::min)(static_cast(buf.size()), size - pos)); + if (inFile->Read(pos, buf.data(), towrite) != towrite) + { + Server->Log("Error reading from in file", LL_ERROR); + exit(2); + } + + if (inParentFile->Read(pos, buf_prev.data(), towrite) != towrite) + { + Server->Log("Error reading from in parent file", LL_ERROR); + exit(2); + } + + if (memcmp(buf.data(), buf_prev.data(), towrite) == 0) + { + continue; + } + + vhdfile->Seek(pos); + if (vhdfile->Write(buf.data(), towrite) != towrite) + { + Server->Log("Error writing to vhd file", LL_ERROR); + exit(2); + } + + written += towrite; + } + + delete vhdfile; + Server->Log("VHDMakeDiff complete. "+PrettyPrintBytes(written)+" written.", LL_INFO); + exit(0); + } std::string hashfilecomp_1=Server->getServerParameter("hashfilecomp_1"); if(!hashfilecomp_1.empty()) diff --git a/fsimageplugin/fsimageplugin.vcxproj b/fsimageplugin/fsimageplugin.vcxproj index c87420bcc..b30d29405 100644 --- a/fsimageplugin/fsimageplugin.vcxproj +++ b/fsimageplugin/fsimageplugin.vcxproj @@ -1,6 +1,10 @@  + + Debug + ARM64 + Debug Win32 @@ -9,6 +13,10 @@ Debug x64 + + Release + ARM64 + Release Win32 @@ -29,23 +37,34 @@ DynamicLibrary Unicode true - v142 + v143 DynamicLibrary Unicode - v142 + v143 DynamicLibrary Unicode true - v142 + v143 + + + DynamicLibrary + Unicode + true + v143 DynamicLibrary Unicode - v142 + v143 + + + DynamicLibrary + Unicode + v143 @@ -59,9 +78,15 @@ + + + + + + <_ProjectFileVersion>10.0.30319.1 @@ -72,23 +97,53 @@ $(Configuration)\ false $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ $(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ true + true $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ $(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ false + false AllRules.ruleset AllRules.ruleset + AllRules.ruleset + + AllRules.ruleset AllRules.ruleset + AllRules.ruleset + + + + + true + + + x64-windows-static-md + + + arm64-windows-static-md + + + x64-windows-static-md + + + x64-windows-static-md + + + x86-windows-static-md @@ -129,8 +184,9 @@ true true MachineX86 - ws2_32.lib;libzstd_x86.lib;%(AdditionalDependencies) - $(SolutionDir)/deps/libs;$(ZstdLibDir) + ws2_32.lib;%(AdditionalDependencies) + + @@ -147,13 +203,34 @@ Level3 ProgramDatabase - $(SolutionDir)/deps/include/imdisk;$(ImdiskIncludeDir);$(ZstdIncludeDir);$(SolutionDir)/deps/include/zstd + $(SolutionDir)/deps/include/imdisk;$(ImdiskIncludeDir) true Windows MachineX64 - ws2_32.lib;libzstd_x86_64.lib;%(AdditionalDependencies) + ws2_32.lib;%(AdditionalDependencies) + $(SolutionDir)/deps/libs;$(ZstdLibDir) + + + + + + Disabled + WIN32;_DEBUG;_WINDOWS;_USRDLL;FSIMAGEPLUGIN_EXPORTS;DO_NOT_USE_CRYPTOPP_MD5;DO_NOT_USE_CRYPTOPP_SHA;%(PreprocessorDefinitions) + true + EnableFastChecks + MultiThreadedDebugDLL + + + Level3 + ProgramDatabase + $(SolutionDir)/deps/include/imdisk;$(ImdiskIncludeDir) + + + true + Windows + ws2_32.lib;%(AdditionalDependencies) $(SolutionDir)/deps/libs;$(ZstdLibDir) @@ -179,8 +256,33 @@ true true MachineX64 - ws2_32.lib;libzstd_x86_64.lib;%(AdditionalDependencies) - $(SolutionDir)/deps/libs;$(ZstdLibDir) + ws2_32.lib;%(AdditionalDependencies) + + + + + + + + MaxSpeed + true + WIN32;NDEBUG;_WINDOWS;_USRDLL;FSIMAGEPLUGIN_EXPORTS;DO_NOT_USE_CRYPTOPP_MD5;DO_NOT_USE_CRYPTOPP_SHA;%(PreprocessorDefinitions) + MultiThreadedDLL + true + + + Level3 + ProgramDatabase + $(SolutionDir)/deps/include/imdisk;$(ImdiskIncludeDir);$(ZstdIncludeDir);$(SolutionDir)/deps/include/zstd + + + true + Windows + true + true + ws2_32.lib;%(AdditionalDependencies) + + @@ -204,6 +306,7 @@ + @@ -226,6 +329,7 @@ + diff --git a/fsimageplugin/partclone.cpp b/fsimageplugin/partclone.cpp index d3077f701..d03b62c2f 100644 --- a/fsimageplugin/partclone.cpp +++ b/fsimageplugin/partclone.cpp @@ -155,9 +155,9 @@ void Partclone::init() Server->Log("Detected fs type "+fstype); if(fstype!="ext2" && fstype!="ext3" - && fstype!="ext4" && fstype!="xfs") + && fstype!="ext4" && fstype!="xfs" && fstype!="btrfs") { - Server->Log("Fs type not supported"); + Server->Log("Fs type not supported: "+fstype); has_error = true; return; } diff --git a/fsimageplugin/vhdfile.cpp b/fsimageplugin/vhdfile.cpp index 36856c67f..c2bc4a865 100644 --- a/fsimageplugin/vhdfile.cpp +++ b/fsimageplugin/vhdfile.cpp @@ -43,34 +43,31 @@ const int64 unixtime_offset=946684800; const unsigned int sector_size=512; -namespace +size_t VHDFile::getNumCompThreads(bool read_only) { - size_t getNumCompThreads(bool read_only) - { - if (read_only) - return 1; + if (read_only) + return 1; - const size_t maxCpus = 5; + const size_t maxCpus = 5; #ifdef _WIN32 - SYSTEM_INFO system_info; - GetSystemInfo(&system_info); - DWORD numCpus = system_info.dwNumberOfProcessors; - if (numCpus == 0) - return 1; - return (std::min)(static_cast(numCpus), maxCpus); + SYSTEM_INFO system_info; + GetSystemInfo(&system_info); + DWORD numCpus = system_info.dwNumberOfProcessors; + if (numCpus == 0) + return 1; + return (std::min)(static_cast(numCpus), maxCpus); #else - long numCpus = sysconf(_SC_NPROCESSORS_ONLN); - if (numCpus < 0) - { - numCpus = 2; - } - else if (numCpus == 0) - { - numCpus = 1; - } - return (std::min)(static_cast(numCpus), maxCpus); -#endif + long numCpus = sysconf(_SC_NPROCESSORS_ONLN); + if (numCpus < 0) + { + numCpus = 2; } + else if (numCpus == 0) + { + numCpus = 1; + } + return (std::min)(static_cast(numCpus), maxCpus); +#endif } VHDFile::VHDFile(const std::string &fn, bool pRead_only, uint64 pDstsize, unsigned int pBlocksize, bool fast_mode, bool compress, size_t compress_n_threads) @@ -1182,12 +1179,13 @@ uint64 VHDFile::getRealSize(void) uint64 VHDFile::usedSize(void) { - uint64 offset_backup=curr_offset; + const uint64 offset_backup=curr_offset; uint64 used_size=0; for(uint64 i=0;i. +**************************************************************************/ + +#include "vhdxfile.h" +#include "../stringtools.h" +#include +#include +#include "CompressedFile.h" +#include "../urbackupcommon/os_functions.h" +#include "FileWrapper.h" +#include "ClientBitmap.h" +#include "IFilesystem.h" +#include "fs/ntfs.h" +#include "vhdfile.h" + +#define PAYLOAD_BLOCK_NOT_PRESENT 0 +#define PAYLOAD_BLOCK_UNDEFINED 1 +#define PAYLOAD_BLOCK_ZERO 2 +#define PAYLOAD_BLOCK_UNMAPPED 3 +#define PAYLOAD_BLOCK_FULLY_PRESENT 6 +#define PAYLOAD_BLOCK_PARTIALLY_PRESENT 7 + +namespace +{ + const int64 vhdx_header_length = 3 * 1024 * 1024; + const int64 allocate_size_add_size = 100 * 1024 * 1024; + const _u32 log_sector_size = 4096; + + size_t roundUp(size_t numToRound, size_t multiple) + { + return ((numToRound + multiple - 1) / multiple) * multiple; + } + + std::vector getFileIdentifier() + { + std::vector ret; + ret.resize(500); + memcpy(ret.data(), "vhdxfile", 8); + std::string creator = Server->ConvertToUTF16("UrBackup vhdx file"); + memcpy(ret.data() + 8, creator.data(), creator.size()); + return ret; + } + + void secureRandomGuid(VhdxGUID& g) + { + Server->secureRandomFill(g, 16); + g[6] = 0x40 | (g[6] & 0xf); + g[8] = 0x80 | (g[8] & 0x3f); + } + + void randomGuid(VhdxGUID& g) + { + Server->randomFill(g, 16); + g[6] = 0x40 | (g[6] & 0xf); + g[8] = 0x80 | (g[8] & 0x3f); + } + + void zeroGUID(VhdxGUID& g) + { + memset(g, 0, 16); + } + + bool equalsGUID(const VhdxGUID& a, const VhdxGUID& b) + { + return memcmp(a, b, sizeof(VhdxGUID)) == 0; + } + + bool isZeroGUID(VhdxGUID& g) + { + VhdxGUID z = {}; + return equalsGUID(g, z); + } + + void copyGUID(const VhdxGUID& src, VhdxGUID& dst) + { + memcpy(dst, src, 16); + } + + void reorderGUID(VhdxGUID& g) + { + *reinterpret_cast(&g[0]) = big_endian(*reinterpret_cast(&g[0])); + *reinterpret_cast(&g[4]) = big_endian(*reinterpret_cast(&g[4])); + *reinterpret_cast(&g[6]) = big_endian(*reinterpret_cast(&g[6])); + } + + bool parseStrGuid(const std::string& str, VhdxGUID& g) + { + if (str.size() < 5) + return false; + + if (str[0] != '{' || str[str.size() - 1] != '}') + return false; + + std::string hb; + for (size_t i = 1; i < str.size() - 1; ++i) + { + if(IsHex(str.substr(i, 1) ) ) + hb+=str[i]; + } + + if (hb.size() != 32) + return false; + + for (size_t i = 0; i < hb.size(); i += 2) + { + std::string cb = hb.substr(i, 2); + g[i/2] = static_cast(hexToULong(cb)); + } + + reorderGUID(g); + + return true; + } + + std::string strGUID(const VhdxGUID& g) + { + VhdxGUID tmp; + copyGUID(g, tmp); + reorderGUID(tmp); + + std::string ret = "{"; + + for (size_t i = 0; i < 16; ++i) + { + ret += byteToHex(tmp[i]); + + if (i == 3 || i==5 || i==7 || i==9) + ret += "-"; + } + + return ret + "}"; + } + + unsigned int crc32c(unsigned char* data, size_t data_size) + { + unsigned int crc = 0xFFFFFFFF; + for (size_t i = 0; i < data_size; ++i) + { + unsigned int b = data[i]; + crc = crc ^ b; + for (int j = 7; j >= 0; j--) + { + unsigned int mask = -1 * (crc & 1); + crc = (crc >> 1) ^ (0x82F63B78 & mask); + } + } + return ~crc; + } + + std::vector getVhdxHeader(uint64 SequenceNumber) + { + std::vector ret; + ret.resize(sizeof(VhdxHeader)); + VhdxHeader* vhdxHeader = reinterpret_cast(ret.data()); + memcpy(ret.data(), "head", 4); + vhdxHeader->SequenceNumber = SequenceNumber; + secureRandomGuid(vhdxHeader->FileWriteGuid); + secureRandomGuid(vhdxHeader->DataWriteGuid); + vhdxHeader->Version = 1; + vhdxHeader->LogOffset = 1 * 1024 * 1024; + vhdxHeader->LogLength = 1 * 1024 * 1024; + vhdxHeader->Checksum = crc32c(reinterpret_cast(&ret[0]), ret.size()); + return ret; + } + + bool checkHeader(IFile* backing_file, VhdxHeader& header) + { + std::string ident(reinterpret_cast(&header), 4); + if (ident != "head") + { + Server->Log("VHDX header tag wrong", LL_WARNING); + return false; + } + + _u32 ccrc = header.Checksum; + + header.Checksum = 0; + + if (crc32c(reinterpret_cast(&header), sizeof(header)) != ccrc) + { + header.Checksum = ccrc; + Server->Log("VHDX header checksum wrong", LL_WARNING); + return false; + } + + header.Checksum = ccrc; + return true; + } + +#pragma pack(1) + struct VhdxRegionTableHeader + { + _u32 Signature; + _u32 Checksum; + _u32 EntryCount; + _u32 Reserved; + }; +#pragma pack() + + int64 getDataBlocks(int64 rawf_size, _u32 block_size) + { + int64 data_blocks = rawf_size / block_size; + if (rawf_size % block_size != 0) ++data_blocks; + return data_blocks; + } + + _u32 getChunkRatio(_u32 block_size, _u32 sector_size) + { + return static_cast<_u32>((8388608LL * sector_size) / block_size); + } + + _u32 getBatEntries(int64 size, _u32 block_size, _u32 sector_size) + { + int64 data_blocks = getDataBlocks(size, block_size); + return static_cast<_u32>(data_blocks + (data_blocks - 1) / getChunkRatio(block_size, sector_size)); + } + + _u32 getBatEntry(int64 pos, _u32 block_size, _u32 sector_size) + { + int64 data_blocks = pos / block_size; + return static_cast<_u32>(data_blocks + data_blocks / getChunkRatio(block_size, sector_size)); + } + + _u32 getSectorBitmapEntry(int64 pos, _u32 block_size, _u32 sector_size) + { + int64 data_blocks = pos / block_size; + _u32 chunk_ratio = getChunkRatio(block_size, sector_size); + return static_cast<_u32>(data_blocks + data_blocks / chunk_ratio + + (chunk_ratio - data_blocks%chunk_ratio)); + } + + _u32 getSectorBitmapOffset(int64 pos, _u32 block_size, _u32 sector_size) + { + int64 sector = pos / sector_size; + return static_cast<_u32>(sector % 8388608LL); + } + + bool isSectorSetInt(const char* sector_buf, + int64 pos, _u32 block_size, _u32 sector_size) + { + _u32 offs = getSectorBitmapOffset(pos, block_size, sector_size); + const char* byte = sector_buf + offs / 8; + _u32 bitmap_bit = offs % 8; + + bool has_bit = (( (*byte) & (1 << bitmap_bit)) > 0); + + return has_bit; + } + + void setSectorInt(char* sector_buf, + int64 start, int64 end, _u32 block_size, _u32 sector_size) + { + while (start < end) + { + _u32 offs = getSectorBitmapOffset(start, block_size, sector_size); + char* byte = sector_buf + offs / 8; + _u32 bitmap_bit = offs % 8; + + *byte = *byte | (1 << bitmap_bit); + start += sector_size; + } + } + + _u32 getBatLength(int64 rawf_size, _u32 block_size, _u32 sector_size) + { + int64 bat_entries = getBatEntries(rawf_size, block_size, sector_size); + + int64 mb_blocks = (bat_entries * sizeof(uint64)) / block_size; + if (bat_entries % block_size != 0) ++mb_blocks; + + return static_cast<_u32>(mb_blocks * 1024 * 1024); + } + + _u32 getSectorBitmapBlocksLength(int64 rawf_size, _u32 block_size, _u32 sector_size) + { + int64 data_blocks = getDataBlocks(rawf_size, block_size); + _u32 chunk_ratio = getChunkRatio(block_size, sector_size); + int64 sector_bitmaps = data_blocks / chunk_ratio; + if (data_blocks % chunk_ratio != 0)++sector_bitmaps; + + return static_cast<_u32>(sector_bitmaps * 1 * 1024 * 1024); + } + + int64 getMetadataSizeSize(int64 rawf_size, _u32 block_size, _u32 sector_size) + { + int64 data_blocks = getDataBlocks(rawf_size, block_size); + _u32 chunk_ratio = getChunkRatio(block_size, sector_size); + int64 sector_bitmaps = data_blocks / chunk_ratio; + if (data_blocks % chunk_ratio != 0)++sector_bitmaps; + + // | -- HEADER -- | -- DATA BLOCKS -- | -- BAT -- | -- SECTOR BITMAP BLOCKS -- | + + return vhdx_header_length + data_blocks * block_size + + getBatLength(rawf_size, block_size, sector_size) + getSectorBitmapBlocksLength(rawf_size, block_size, sector_size); + } + + void makeMetaTableGUID(VhdxGUID& g) + { + unsigned char meta_guid[16] = { 0x8B, 0x7C, 0xA2, 0x06, 0x47, 0x90, 0x4B, 0x9A, 0xB8, 0xFE, 0x57, 0x5F, 0x05, 0x0F, 0x88, 0x6E }; + memcpy(g, meta_guid, sizeof(meta_guid)); + reorderGUID(g); + } + + void makeBatGUID(VhdxGUID& g) + { + unsigned char bat_guid[16] = { 0x2D, 0xC2, 0x77, 0x66, 0xF6, 0x23, 0x42, 0x00, 0x9D, 0x64, 0x11, 0x5E, 0x9B, 0xFD, 0x4A, 0x08 }; + memcpy(g, bat_guid, sizeof(bat_guid)); + reorderGUID(g); + } + + const uint64 meta_region_offset = 2 * 1024 * 1024; + const uint64 bat_table_offset = meta_region_offset + 1 * 1024 * 1024; + + std::vector getVhdxRegionTable(int64 rawf_size, _u32 block_size, _u32 sector_size) + { + std::vector ret; + ret.resize(64 * 1024); + + memcpy(ret.data(), "regi", 4); + VhdxRegionTableHeader* header = reinterpret_cast(ret.data()); + header->EntryCount = 2; + + VhdxRegionTableEntry* meta_entry = reinterpret_cast(ret.data() + sizeof(VhdxRegionTableHeader)); + makeMetaTableGUID(meta_entry->Guid); + meta_entry->FileOffset = meta_region_offset; + meta_entry->Length = 1 * 1024 * 1024; + meta_entry->Required = 1; + + VhdxRegionTableEntry* bat_entry = reinterpret_cast(ret.data() + sizeof(VhdxRegionTableHeader) + + sizeof(VhdxRegionTableEntry)); + makeBatGUID(bat_entry->Guid); + bat_entry->FileOffset = bat_table_offset; + bat_entry->Length = getBatLength(rawf_size, block_size, sector_size); + bat_entry->Required = 1; + + header->Checksum = crc32c(reinterpret_cast(&ret[0]), ret.size()); + + return ret; + } + +#pragma pack(1) + struct VhdxMetadataTableHeader + { + uint64 Signature; + unsigned short Reserved; + unsigned short EntryCount; + _u32 Reserved2[5]; + }; + + struct VhdxMetadataTableEntry + { + VhdxGUID ItemId; + _u32 Offset; + _u32 Length; + _u32 IsUser : 1; + _u32 IsVirtualDisk : 1; + _u32 IsRequired : 1; + _u32 Reserved : 29; + _u32 Reserved2; + }; + + struct VhdxVirtualDiskSize + { + uint64 VirtualDiskSize; + }; + + struct VhdxVirtualDiskLogicalSectorSize + { + _u32 LogicalSectorSize; + }; + + struct VhdxPhysicalDiskSectorSize + { + _u32 PhysicalSectorSize; + }; + + struct VhdxVirtualDiskId + { + VhdxGUID VirtualDiskId; + }; + + struct VhdxParentLocatorHeader + { + VhdxGUID LocatorType; + unsigned short Reserved; + unsigned short KeyValueCount; + }; + + struct VhdxParentLocatorEntry + { + _u32 KeyOffset; + _u32 ValueOffset; + unsigned short KeyLength; + unsigned short ValueLength; + }; + +#pragma pack() + + void makeFileParametersGUID(VhdxGUID& g) + { + unsigned char file_parameters_guid[16] = { 0xCA, 0xA1, 0x67, 0x37, 0xFA, 0x36, 0x4D, 0x43, 0xB3, 0xB6, 0x33, 0xF0, 0xAA, 0x44, 0xE7, 0x6B }; + memcpy(&g, file_parameters_guid, sizeof(file_parameters_guid)); + reorderGUID(g); + } + + void makeVirtualDiskSizeGUID(VhdxGUID& g) + { + unsigned char virtual_disk_size_guid[16] = { 0x2F, 0xA5, 0x42, 0x24, 0xCD, 0x1B, 0x48, 0x76, 0xB2, 0x11, 0x5D, 0xBE, 0xD8, 0x3B, 0xF4, 0xB8 }; + memcpy(&g, virtual_disk_size_guid, sizeof(virtual_disk_size_guid)); + reorderGUID(g); + } + + void makeLogicalSectorSizeGUID(VhdxGUID& g) + { + unsigned char logical_sector_size_guid[16] = { 0x81, 0x41, 0xBF, 0x1D, 0xA9, 0x6F, 0x47, 0x09, 0xBA, 0x47, 0xF2, 0x33, 0xA8, 0xFA, 0xAB, 0x5F }; + memcpy(&g, logical_sector_size_guid, sizeof(logical_sector_size_guid)); + reorderGUID(g); + } + + void makePhysicalSectorSizeGUID(VhdxGUID& g) + { + unsigned char physical_sector_size_guid[16] = { 0xCD, 0xA3, 0x48, 0xC7, 0x44, 0x5D, 0x44, 0x71, 0x9C, 0xC9, 0xE9, 0x88, 0x52, 0x51, 0xC5, 0x56 }; + memcpy(&g, physical_sector_size_guid, sizeof(physical_sector_size_guid)); + reorderGUID(g); + } + + void makeVirtualDiskIdGUID(VhdxGUID& g) + { + unsigned char page83_data_guid[16] = { 0xBE, 0xCA, 0x12, 0xAB, 0xB2, 0xE6, 0x45, 0x23, 0x93, 0xEF, 0xC3, 0x09, 0xE0, 0x00, 0xC7, 0x46 }; + memcpy(&g, page83_data_guid, sizeof(page83_data_guid)); + reorderGUID(g); + } + + void makeParentLocatorGUID(VhdxGUID& g) + { + unsigned char parent_locator_guid[16] = { 0xA8, 0xD3, 0x5F, 0x2D, 0xB3, 0x0B, 0x45, 0x4D, 0xAB, 0xF7, 0xD3, 0xD8, 0x48, 0x34, 0xAB, 0x0C }; + memcpy(&g, parent_locator_guid, sizeof(parent_locator_guid)); + reorderGUID(g); + } + + void makeVhdxParentLocatorGUID(VhdxGUID& g) + { + unsigned char vhdx_parent_locator_guid[16] = { 0xB0, 0x4A, 0xEF, 0xB7, 0xD1, 0x9E, 0x4A, 0x81, 0xB7, 0x89, 0x25, + 0xB8, 0xE9, 0x44, 0x59, 0x13 }; + memcpy(&g, vhdx_parent_locator_guid, sizeof(vhdx_parent_locator_guid)); + reorderGUID(g); + } + + std::vector getMetaRegion(int64 rawf_size, _u32 block_size, _u32 sector_size, + std::string parent_data_uuid, std::string parent_rel_loc, std::string parent_abs_loc) + { + size_t parent_locator_size = 0; + + str_map parent_loc_entries; + + if (!parent_data_uuid.empty()) + { + parent_loc_entries[Server->ConvertToUTF16("parent_linkage")] = Server->ConvertToUTF16(parent_data_uuid); + parent_loc_entries[Server->ConvertToUTF16("relative_path")] = Server->ConvertToUTF16(parent_rel_loc); + parent_loc_entries[Server->ConvertToUTF16("absolute_win32_path")] = Server->ConvertToUTF16(parent_abs_loc); + + parent_locator_size = sizeof(VhdxParentLocatorHeader); + parent_locator_size += sizeof(VhdxParentLocatorEntry) * parent_loc_entries.size(); + + for (str_map::iterator it = parent_loc_entries.begin(); + it!=parent_loc_entries.end();++it) + { + parent_locator_size += it->first.size(); + parent_locator_size += it->second.size(); + } + } + + std::vector ret; + ret.resize(64 * 1024 + + sizeof(VhdxFileParameters) + + sizeof(VhdxVirtualDiskSize) + + sizeof(VhdxVirtualDiskLogicalSectorSize) + + sizeof(VhdxPhysicalDiskSectorSize) + + sizeof(VhdxVirtualDiskId) + + parent_locator_size); + + memcpy(ret.data(), "metadata", 8); + VhdxMetadataTableHeader* header = reinterpret_cast(ret.data()); + header->EntryCount = 5; + + VhdxMetadataTableEntry* file_parameters_entry = reinterpret_cast(ret.data() + sizeof(VhdxMetadataTableHeader)); + makeFileParametersGUID(file_parameters_entry->ItemId); + file_parameters_entry->Offset = 64 * 1024; + file_parameters_entry->Length = sizeof(VhdxFileParameters); + file_parameters_entry->IsRequired = 1; + + VhdxFileParameters* file_parameters = reinterpret_cast(ret.data() + file_parameters_entry->Offset); + file_parameters->BlockSize = block_size; + file_parameters->LeaveBlocksAllocated = 0; + file_parameters->HasParent = parent_data_uuid.empty() ? 0 : 1; + + VhdxMetadataTableEntry* virtual_disk_size_entry = reinterpret_cast(ret.data() + sizeof(VhdxMetadataTableHeader) + + sizeof(VhdxMetadataTableEntry)); + makeVirtualDiskSizeGUID(virtual_disk_size_entry->ItemId); + virtual_disk_size_entry->Offset = 64 * 1024 + sizeof(VhdxFileParameters); + virtual_disk_size_entry->Length = sizeof(VhdxFileParameters); + virtual_disk_size_entry->IsRequired = 1; + virtual_disk_size_entry->IsVirtualDisk = 1; + + VhdxVirtualDiskSize* virtual_disk_size = reinterpret_cast(ret.data() + virtual_disk_size_entry->Offset); + virtual_disk_size->VirtualDiskSize = rawf_size; + + VhdxMetadataTableEntry* logical_sector_size_entry = reinterpret_cast(ret.data() + sizeof(VhdxMetadataTableHeader) + + 2 * sizeof(VhdxMetadataTableEntry)); + makeLogicalSectorSizeGUID(logical_sector_size_entry->ItemId); + logical_sector_size_entry->Offset = 64 * 1024 + sizeof(VhdxFileParameters) + sizeof(VhdxVirtualDiskSize); + logical_sector_size_entry->Length = sizeof(VhdxVirtualDiskLogicalSectorSize); + logical_sector_size_entry->IsRequired = 1; + logical_sector_size_entry->IsVirtualDisk = 1; + + VhdxVirtualDiskLogicalSectorSize* logical_sector_size = reinterpret_cast(ret.data() + logical_sector_size_entry->Offset); + logical_sector_size->LogicalSectorSize = sector_size; + + VhdxMetadataTableEntry* physical_sector_size_entry = reinterpret_cast(ret.data() + sizeof(VhdxMetadataTableHeader) + + 3 * sizeof(VhdxMetadataTableEntry)); + makePhysicalSectorSizeGUID(physical_sector_size_entry->ItemId); + physical_sector_size_entry->Offset = 64 * 1024 + sizeof(VhdxFileParameters) + sizeof(VhdxVirtualDiskSize) + sizeof(VhdxVirtualDiskLogicalSectorSize); + physical_sector_size_entry->Length = sizeof(VhdxPhysicalDiskSectorSize); + physical_sector_size_entry->IsRequired = 1; + physical_sector_size_entry->IsVirtualDisk = 1; + + VhdxPhysicalDiskSectorSize* physical_sector_size = reinterpret_cast(ret.data() + physical_sector_size_entry->Offset); + physical_sector_size->PhysicalSectorSize = sector_size; + + VhdxMetadataTableEntry* page83_data_entry = reinterpret_cast(ret.data() + sizeof(VhdxMetadataTableHeader) + + 4 * sizeof(VhdxMetadataTableEntry)); + makeVirtualDiskIdGUID(page83_data_entry->ItemId); + page83_data_entry->Offset = 64 * 1024 + sizeof(VhdxFileParameters) + sizeof(VhdxVirtualDiskSize) + sizeof(VhdxVirtualDiskLogicalSectorSize) + sizeof(VhdxPhysicalDiskSectorSize); + page83_data_entry->Length = sizeof(VhdxVirtualDiskId); + page83_data_entry->IsRequired = 1; + page83_data_entry->IsVirtualDisk = 1; + + VhdxVirtualDiskId* virtual_disk_id = reinterpret_cast(ret.data() + page83_data_entry->Offset); + secureRandomGuid(virtual_disk_id->VirtualDiskId); + + if (!parent_data_uuid.empty()) + { + ++header->EntryCount; + + VhdxMetadataTableEntry* parent_locator_entry = reinterpret_cast(ret.data() + sizeof(VhdxMetadataTableHeader) + + 5 * sizeof(VhdxMetadataTableEntry)); + makeParentLocatorGUID(parent_locator_entry->ItemId); + parent_locator_entry->Offset = 64 * 1024 + sizeof(VhdxFileParameters) + sizeof(VhdxVirtualDiskSize) + + sizeof(VhdxVirtualDiskLogicalSectorSize) + sizeof(VhdxPhysicalDiskSectorSize) + sizeof(VhdxVirtualDiskId); + parent_locator_entry->Length = static_cast<_u32>(parent_locator_size); + parent_locator_entry->IsRequired = 1; + + VhdxParentLocatorHeader* parent_locator_header = reinterpret_cast(ret.data() + parent_locator_entry->Offset); + + parent_locator_header->KeyValueCount = static_cast<_u16>(parent_loc_entries.size()); + makeVhdxParentLocatorGUID(parent_locator_header->LocatorType); + + size_t entry_pos = parent_locator_entry->Offset + sizeof(VhdxParentLocatorHeader); + size_t str_pos = parent_locator_entry->Offset + sizeof(VhdxParentLocatorHeader) + + sizeof(VhdxParentLocatorEntry) * parent_loc_entries.size(); + + for (str_map::iterator it = parent_loc_entries.begin(); + it!=parent_loc_entries.end();++it) + { + VhdxParentLocatorEntry* entry = reinterpret_cast(ret.data() + entry_pos); + entry_pos += sizeof(VhdxParentLocatorEntry); + + entry->KeyOffset = static_cast<_u32>(str_pos - parent_locator_entry->Offset); + entry->KeyLength = static_cast<_u16>(it->first.size()); + memcpy(ret.data() + str_pos, it->first.data(), it->first.size()); + str_pos += it->first.size(); + + entry->ValueOffset = static_cast<_u32>(str_pos - parent_locator_entry->Offset); + entry->ValueLength = static_cast<_u16>(it->second.size()); + memcpy(ret.data() + str_pos, it->second.data(), it->second.size()); + str_pos += it->second.size(); + } + + assert(str_pos == ret.size()); + } + + return ret; + } + +#pragma pack(1) + struct LogEntryHeader + { + _u32 signature; + _u32 Checksum; + _u32 EntryLength; + _u32 Tail; + int64 SequenceNumber; + _u32 DescriptorCount; + _u32 Reserved; + VhdxGUID LogGuid; + int64 FlushedFileOffset; + int64 LastFileOffset; + }; + + struct LogZeroDescriptor + { + _u32 signature; + _u32 Reserved; + int64 ZeroLength; + int64 FileOffset; + int64 SequenceNumber; + }; + + struct LogDataDescriptor + { + _u32 signature; + char TrailingBytes[4]; + char LeadingBytes[8]; + int64 FileOffset; + int64 SequenceNumber; + }; + + struct LogDataSector + { + _u32 signature; + _u32 SequenceHigh; + char data[4084]; + _u32 SequenceLow; + }; +#pragma pack() + + + struct LogData + { + int64 offset; + char data[4096]; + }; + + struct LogEntry + { + LogEntry() + : sequence_number(-1) {} + + std::vector to_zero; + std::vector to_write; + int64 sequence_number; + int64 length; + int64 fsize; + int64 new_fsize; + uint64 tail_pos; + }; + + typedef union + { + struct { + _u32 LowPart; + _u32 HighPart; + }; + struct { + _u32 LowPart; + _u32 HighPart; + } u; + int64 QuadPart; + } SSequence; + + LogEntry readLogEntry(IFile* f, const VhdxGUID& log_guid, int64 off) + { + LogEntry loge; + + std::vector buf(4096); + + if (f->Read(off, buf.data(), static_cast<_u32>(buf.size())) != buf.size()) + { + Server->Log("Error reading log entry header. " + os_last_error_str(), LL_WARNING); + return loge; + } + + std::string signature(buf.data(), 4); + + if (signature != "loge") + return loge; + + LogEntryHeader* header = reinterpret_cast(buf.data()); + + if (!equalsGUID(header->LogGuid, log_guid)) + return loge; + + loge.length = header->EntryLength; + std::vector entry_buf(header->EntryLength); + + if (f->Read(off, entry_buf.data(), static_cast<_u32>(entry_buf.size())) != entry_buf.size()) + { + Server->Log("Error reading log entry (size=" + convert(header->EntryLength) + "). " + + os_last_error_str(), LL_WARNING); + return loge; + } + + _u32 checksum = header->Checksum; + + header = reinterpret_cast(entry_buf.data()); + header->Checksum = 0; + + _u32 checksum_calc = crc32c(reinterpret_cast(entry_buf.data()), entry_buf.size()); + + if (checksum_calc != checksum) + { + Server->Log("Log entry checksum is wrong", LL_WARNING); + return loge; + } + + int64 entry_seq = header->SequenceNumber; + + loge.fsize = header->FlushedFileOffset; + loge.new_fsize = header->LastFileOffset; + loge.tail_pos = header->Tail; + + int64 desc_off = 4096; + if (header->DescriptorCount > 126) + { + desc_off += ((header->DescriptorCount - 126) / 128 ) *4096; + if ( (header->DescriptorCount - 126) % 128 != 0) + desc_off += 4096; + } + + for (int64 i = 0; i < header->DescriptorCount; ++i) + { + char* desc_ptr = entry_buf.data() + 64 + i * 32; + std::string desc_sig(desc_ptr, 4); + + if (desc_sig == "zero") + { + LogZeroDescriptor* zero_desc = reinterpret_cast(desc_ptr); + if (entry_seq != zero_desc->SequenceNumber) + { + Server->Log("Zero log entry sequence number is wrong", LL_WARNING); + return loge; + } + + loge.to_zero.push_back(*zero_desc); + } + else if (desc_sig == "desc") + { + LogDataDescriptor* data_desc = reinterpret_cast(desc_ptr); + if (entry_seq != data_desc->SequenceNumber) + { + Server->Log("Data log entry sequence number is wrong", LL_WARNING); + return loge; + } + + LogDataSector* data_sec = reinterpret_cast(entry_buf.data() + desc_off); + + std::string data_sec_sig(entry_buf.data() + desc_off, 4); + + if (data_sec_sig != "data") + { + Server->Log("Data log entry signature is wrong", LL_WARNING); + return loge; + } + + SSequence seq; + seq.QuadPart = data_desc->SequenceNumber; + + if (data_sec->SequenceHigh != seq.HighPart) + { + Server->Log("Data log entry high sequence number is wrong", LL_WARNING); + return loge; + } + + if (data_sec->SequenceLow != seq.LowPart) + { + Server->Log("Data log entry low sequence number is wrong", LL_WARNING); + return loge; + } + + LogData log_data; + log_data.offset = data_desc->FileOffset; + memcpy(log_data.data, data_desc->LeadingBytes, 8); + memcpy(log_data.data + 8, data_sec->data, sizeof(data_sec->data)); + memcpy(log_data.data + 8 + sizeof(data_sec->data), data_desc->TrailingBytes, 4); + + loge.to_write.push_back(log_data); + } + else + { + Server->Log("Unknown log entry signature", LL_WARNING); + return loge; + } + } + + loge.sequence_number = entry_seq; + return loge; + } +} + +VHDXFile::VHDXFile(const std::string& fn, bool pRead_only, uint64 pDstsize, unsigned int pBlocksize, bool fast_mode, bool compress, size_t compress_n_threads) + : dst_size(pDstsize), fast_mode(fast_mode), read_only(pRead_only), sector_bitmap_mutex(Server->createMutex()), + pending_sector_bitmaps_mutex(Server->createMutex()), finished(false), + log_sequence_num(1), spos(0), data_write_uuid_updated(false), + is_open(false), next_payload_mutex(Server->createMutex()), log_mutex(Server->createMutex()) +{ + is_open = open(fn, compress, compress_n_threads); +} + +VHDXFile::VHDXFile(const std::string& fn, const std::string& parent_fn, bool pRead_only, + bool fast_mode, bool compress, uint64 pDstsize, size_t compress_n_threads) + : fast_mode(fast_mode), read_only(pRead_only), dst_size(pDstsize), + parent_fn(parent_fn), sector_bitmap_mutex(Server->createMutex()), + pending_sector_bitmaps_mutex(Server->createMutex()), finished(false), + log_sequence_num(1), spos(0), data_write_uuid_updated(false), + is_open(false), next_payload_mutex(Server->createMutex()), log_mutex(Server->createMutex()) +{ + if (!FileExists(fn)) + { + parent.reset(new VHDXFile(parent_fn, true, + 0)); + + if (!parent->isOpen()) + { + Server->Log("Error opening VHDX parent at \"" + parent_fn + "\"", LL_ERROR); + return; + } + + dst_size = parent->getSize(); + + if (pDstsize > 0 && pDstsize != dst_size) + { + dst_size = pDstsize; + } + } + + is_open = open(fn, compress, compress_n_threads); +} + +VHDXFile::~VHDXFile() +{ + if (!is_open) + return; + + if (!read_only) + { + finish(); + } +} + +bool VHDXFile::Seek(_i64 offset) +{ + spos = offset; + return true; +} + +bool VHDXFile::Read(char* buffer, size_t bsize, size_t& read) +{ + bool has_read_error = false; + read = Read(spos, buffer, static_cast<_u32>(bsize), &has_read_error); + spos += read; + return !has_read_error; +} + +_u32 VHDXFile::Write(const char* buffer, _u32 bsize, bool* has_error) +{ + _u32 rc = Write(spos, buffer, bsize, has_error); + spos += rc; + return rc; +} + +bool VHDXFile::isOpen(void) +{ + return is_open; +} + +uint64 VHDXFile::getSize(void) +{ + return Size(); +} + +uint64 VHDXFile::usedSize() +{ + const int64 spos_backup = spos; + + uint64 ret = 0; + for (int64 i = 0; i < dst_size; i += block_size) + { + spos = i; + if(has_sector()) + ret += block_size; + } + + spos = spos_backup; + + return ret; +} + +std::string VHDXFile::getFilename(void) +{ + return file->getFilename(); +} + +bool VHDXFile::has_sector(_i64 sector_size) +{ + if (!has_sector_int(spos)) + { + if (parent.get() != NULL) + return parent->has_sector_int(spos); + } + + return true; +} + +bool VHDXFile::this_has_sector(_i64 sector_size) +{ + return has_sector_int(spos); +} + +unsigned int VHDXFile::getBlocksize() +{ + return block_size; +} + +bool VHDXFile::finish() +{ + if (!finished) + { + finished = true; + + if (read_only) + return true; + + bool ret = syncInt(true); + + if (ret && parent.get()!=NULL) + { + ret = parent->finish(); + } + + CompressedFile* compfile = dynamic_cast(file); + if (compfile != NULL) + { + if (compfile->finish()) + { + finished = true; + return true; + } + } + } + return true; +} + +bool VHDXFile::trimUnused(_i64 fs_offset, _i64 trim_blocksize, ITrimCallback* trim_callback) +{ + return true; +} + +bool VHDXFile::syncBitmap(_i64 fs_offset) +{ + return true; +} + +bool VHDXFile::makeFull(_i64 fs_offset, IVHDWriteCallback* write_callback) +{ + FileWrapper devfile(this, fs_offset); + std::auto_ptr bitmap_source; + + bitmap_source.reset(new ClientBitmap(backing_file->getFilename() + ".cbitmap")); + + if (bitmap_source->hasError()) + { + Server->Log("Error reading client bitmap. Falling back to reading bitmap from NTFS", LL_WARNING); + + bitmap_source.reset(new FSNTFS(&devfile, IFSImageFactory::EReadaheadMode_None, false, NULL)); + } + + if (bitmap_source->hasError()) + { + Server->Log("Error opening NTFS bitmap. Cannot convert incremental to full image.", LL_WARNING); + return false; + } + + unsigned int bitmap_blocksize = static_cast(bitmap_source->getBlocksize()); + + std::vector buffer; + buffer.resize(sector_size); + + int64 ntfs_blocks_per_vhd_sector = block_size / bitmap_blocksize; + + for (int64 ntfs_block = 0, n_ntfs_blocks = devfile.Size() / bitmap_blocksize; + ntfs_block < n_ntfs_blocks; ntfs_block += ntfs_blocks_per_vhd_sector) + { + bool has_vhd_sector = false; + for (int64 i = ntfs_block; + i < ntfs_block + ntfs_blocks_per_vhd_sector + && i < n_ntfs_blocks; ++i) + { + if (bitmap_source->hasBlock(i)) + { + has_vhd_sector = true; + break; + } + } + + if (has_vhd_sector) + { + int64 block_pos = fs_offset + ntfs_block * bitmap_blocksize; + int64 max_block_pos = (std::min)(fs_offset + ntfs_block * bitmap_blocksize + block_size, + fs_offset + n_ntfs_blocks * bitmap_blocksize); + for (int64 i = block_pos; i < max_block_pos; i += sector_size) + { + Seek(i); + + if (!has_block(false) + && has_block(true)) + { + bool has_error = false; + if (Read(buffer.data(), sector_size) != sector_size) + { + Server->Log("Error converting incremental to full image. Cannot read from parent VHDX file at position " + convert(i), LL_WARNING); + return false; + } + + if (!write_callback->writeVHD(i, buffer.data(), sector_size)) + { + Server->Log("Error converting incremental to full image. Cannot write to VHDX file at position " + convert(i), LL_WARNING); + return false; + } + } + } + } + else + { + int64 block_pos = ntfs_block * bitmap_blocksize; + int64 max_block_pos = (std::min)(ntfs_block * bitmap_blocksize + block_size, + n_ntfs_blocks * bitmap_blocksize); + + write_callback->emptyVHDBlock(block_pos, max_block_pos); + } + } + + parent.reset(); + parent_fn.clear(); + + std::vector meta_region = getMetaRegion(dst_size, block_size, sector_size, + std::string(), std::string(), std::string()); + + if (file->Write(meta_region_offset, meta_region.data(), static_cast<_u32>(meta_region.size())) != meta_region.size()) + return false; + + return true; +} + +bool VHDXFile::setUnused(_i64 unused_start, _i64 unused_end) +{ + if (!Seek(unused_start)) + { + Server->Log("Error while seeking to " + convert(unused_end) + " in VHDX file." + "Size is " + convert(dst_size) + " -2", LL_ERROR); + return false; + } + + if (read_only) + { + Server->Log("VHDX file is read only -2", LL_ERROR); + return false; + } + + if (unused_end > dst_size) + { + Server->Log("VHDX file is not large enough. Want to trim till " + + convert(unused_end) + " but size is " + convert(dst_size), LL_ERROR); + return false; + } + + std::vector zero_buf; + + while (unused_start< unused_end) + { + int64 block = getBatEntry(unused_start, block_size, sector_size); + VhdxBatEntry* bat_entry = reinterpret_cast(bat_buf.data()) + block; + + if (unused_start % block_size == 0 && + unused_start + block_size <= unused_end) + { + bat_entry->State = PAYLOAD_BLOCK_ZERO; + unused_start += block_size; + continue; + } + + _u32 curr_sector_size = sector_size; + if (unused_start % sector_size != 0) + { + curr_sector_size = sector_size - unused_start % sector_size; + } + + size_t wantwrite = (std::min)(static_cast(curr_sector_size), + static_cast(unused_end - unused_start)); + + bool copy_prev = false; + + if (bat_entry->State == PAYLOAD_BLOCK_PARTIALLY_PRESENT) + { + bool set; + if (!isSectorSet(unused_start, set)) + { + return false; + } + + if (!set) + { + if (!setSector(unused_start)) + { + return false; + } + + copy_prev = true; + } + else + { + if (zero_buf.size() != wantwrite) + { + zero_buf.resize(wantwrite); + } + _u32 rc = file->Write(bat_entry->FileOffsetMB * 1024 * 1024 + unused_start % block_size, + zero_buf.data(), static_cast<_u32>(wantwrite)); + } + + unused_start += wantwrite; + } + else if (bat_entry->State == PAYLOAD_BLOCK_UNDEFINED || + bat_entry->State == PAYLOAD_BLOCK_UNMAPPED || + bat_entry->State == PAYLOAD_BLOCK_NOT_PRESENT) + { + if (!allocateBatBlockFull(block)) + { + return false; + } + + bat_entry->State = PAYLOAD_BLOCK_PARTIALLY_PRESENT; + + if (!setSector(unused_start)) + { + return false; + } + + unused_start += wantwrite; + + copy_prev = true; + } + else if (bat_entry->State != PAYLOAD_BLOCK_ZERO) + { + Server->Log("Unknown bat entry state " + convert(bat_entry->State), LL_ERROR); + return false; + } + + if (copy_prev && curr_sector_size < sector_size && + parent.get()!=NULL) + { + std::vector prev_buf(sector_size - curr_sector_size); + + int64 prev_pos = (unused_start / sector_size) * sector_size; + + _u32 rc = parent->Read(prev_pos, + prev_buf.data(), static_cast<_u32>(prev_buf.size())); + + if (rc != prev_buf.size()) + return false; + + rc = file->Write(bat_entry->FileOffsetMB * 1024 * 1024 + prev_pos % block_size, + prev_buf.data(), static_cast<_u32>(prev_buf.size())); + + if (rc != prev_buf.size()) + return false; + } + if (copy_prev && unused_start + wantwrite == unused_end && + unused_end % sector_size != 0) + { + std::vector prev_buf(sector_size - unused_end%sector_size); + + int64 prev_pos = unused_end; + + _u32 rc = parent->Read(prev_pos, + prev_buf.data(), static_cast<_u32>(prev_buf.size())); + + if (rc != prev_buf.size()) + return false; + + rc = file->Write(bat_entry->FileOffsetMB * 1024 * 1024 + prev_pos % block_size, + prev_buf.data(), static_cast<_u32>(prev_buf.size())); + + if (rc != prev_buf.size()) + return false; + } + } + + return true; +} + +bool VHDXFile::setBackingFileSize(_i64 fsize) +{ + fsize += 1 * 1024 * 1024; + fsize += bat_region.Length; + fsize += curr_header.LogLength; + fsize += meta_table_region.Length; + + if (fsize > file->Size()) + { + return file->Resize(fsize, false); + } + + return false; +} + +std::string VHDXFile::Read(_u32 tr, bool* has_error) +{ + std::string ret = Read(spos, tr, has_error); + spos += ret.size(); + return ret; +} + +std::string VHDXFile::Read(int64 spos, _u32 tr, bool* has_error) +{ + std::string ret; + ret.resize(tr); + + _u32 rc = Read(spos, &ret[0], tr, has_error); + if (rc < tr) + ret.resize(rc); + + return ret; +} + +_u32 VHDXFile::Read(char* buffer, _u32 bsize, bool* has_error) +{ + _u32 rc = Read(spos, buffer, bsize, has_error); + spos += rc; + return rc; +} + +_u32 VHDXFile::Read(int64 spos, char* buffer, _u32 bsize, bool* has_error) +{ + if (spos> dst_size) + { + Server->Log("Error reading from VHDX file. Trying to read beyond file size at " + convert(spos) + " size=" + convert(dst_size)); + if (has_error != NULL) + *has_error = true; + + return 0; + } + else if (spos + bsize >= dst_size) + { + bsize = static_cast<_u32>(dst_size - spos); + } + + _u32 read = 0; + while (bsize - read > 0) + { + _u32 block = getBatEntry(spos, block_size, sector_size); + + const VhdxBatEntry* bat_entry = reinterpret_cast(bat_buf.data()) + block; + + if (bat_entry->State == PAYLOAD_BLOCK_FULLY_PRESENT) + { + const _u32 toread = (std::min)(block_size - static_cast<_u32>(spos % block_size), bsize - read); + + const int64 fpos = bat_entry->FileOffsetMB * 1024 * 1024 + spos % block_size; + const _u32 rc = file->Read(fpos, + buffer + read, toread); + + read += rc; + spos += rc; + + if (rc < toread) + { + Server->Log("Error reading " + convert(toread) + " bytes from vhdx file at pos " + convert(fpos) + + " read " + convert(rc) + " toread " + convert(toread) + " error: " + os_last_error_str()); + if (has_error != NULL) + *has_error = true; + + return read; + } + + continue; + } + + if (parent.get()==NULL) + { + _u32 toread = (std::min)(block_size - static_cast<_u32>(spos % block_size), bsize - read); + + if (bat_entry->State == PAYLOAD_BLOCK_PARTIALLY_PRESENT) + { + Server->Log("VHDX parent partially present though there is no parent pos=" + convert(spos), LL_WARNING); + if (has_error != NULL) + *has_error = true; + + return read; + } + else if (bat_entry->State == PAYLOAD_BLOCK_NOT_PRESENT || + bat_entry->State == PAYLOAD_BLOCK_UNDEFINED || + bat_entry->State == PAYLOAD_BLOCK_ZERO || + bat_entry->State == PAYLOAD_BLOCK_UNMAPPED) + { + toread = (std::min)(block_size - static_cast<_u32>(spos % block_size), bsize - read); + + memset(buffer+read, 0, toread); + read += toread; + spos += toread; + } + else + { + Server->Log("Unknown VHDX bat state " + convert(bat_entry->State) + " pos=" + convert(spos), LL_WARNING); + if (has_error != NULL) + *has_error = true; + + return read; + } + } + else + { + _u32 toread; + if (bat_entry->State != PAYLOAD_BLOCK_PARTIALLY_PRESENT) + { + toread = (std::min)(block_size - static_cast<_u32>(spos % block_size), bsize - read); + } + else + { + toread = (std::min)(sector_size - static_cast<_u32>(spos % sector_size), bsize - read); + } + + if (bat_entry->State == PAYLOAD_BLOCK_PARTIALLY_PRESENT) + { + bool set; + if (!isSectorSet(spos, set)) + { + Server->Log("Sector of partially present VHDX block not set pos=" + convert(spos), LL_WARNING); + if (has_error != NULL) + *has_error = true; + + return read; + } + + _u32 rc; + if (set) + { + rc = file->Read(bat_entry->FileOffsetMB * 1024 * 1024 + spos % block_size, + buffer + read, toread); + } + else + { + rc = parent->Read(spos, buffer + read, toread); + } + + read += rc; + spos += rc; + + if (rc < toread) + { + Server->Log("Error reading " + convert(toread) + " bytes from vhdx file at pos " + + convert(bat_entry->FileOffsetMB * 1024 * 1024 + spos % block_size) + " spos " + convert(spos) + " set " + convert(set) + + " read " + convert(rc) + " toread " + convert(toread) + " error: " + os_last_error_str()); + + if (has_error != NULL) + *has_error = true; + + return read; + } + } + else if (bat_entry->State == PAYLOAD_BLOCK_UNDEFINED || + bat_entry->State == PAYLOAD_BLOCK_ZERO || + bat_entry->State == PAYLOAD_BLOCK_UNMAPPED) + { + memset(buffer + read, 0, toread); + read += toread; + spos += toread; + } + else if (bat_entry->State == PAYLOAD_BLOCK_NOT_PRESENT) + { + const _u32 rc = parent->Read(spos, buffer + read, toread, has_error); + + read += rc; + spos += rc; + + if (rc < toread) + { + if (has_error != NULL) + *has_error = true; + + return read; + } + } + else + { + Server->Log("Unknown VHDX bat state (with parent) " + convert(bat_entry->State) + " pos=" + convert(spos), LL_WARNING); + + if (has_error != NULL) + *has_error = true; + + return read; + } + } + } + return read; +} + +_u32 VHDXFile::Write(const std::string& tw, bool* has_error) +{ + _u32 rc = Write(spos, tw.data(), static_cast<_u32>(tw.size()), has_error); + spos += rc; + return rc; +} + +_u32 VHDXFile::Write(int64 spos, const std::string& tw, bool* has_error) +{ + return Write(spos, tw.data(), static_cast<_u32>(tw.size()), has_error); +} + +_u32 VHDXFile::Write(int64 spos, const char* buffer, _u32 bsize, bool* has_error) +{ + if (spos > dst_size) + { + if (has_error != NULL) + *has_error = true; + + return 0; + } + else if (spos + bsize >= dst_size) + { + bsize = static_cast<_u32>(dst_size - spos); + } + + if (!data_write_uuid_updated) + { + randomGuid(curr_header.DataWriteGuid); + + data_write_uuid_updated = true; + + if (!fast_mode && !updateHeader()) + { + if (has_error != NULL) + *has_error = true; + + return 0; + } + } + + _u32 written = 0; + while (bsize - written > 0) + { + int64 block = getBatEntry(spos, block_size, sector_size); + VhdxBatEntry* bat_entry = reinterpret_cast(bat_buf.data()) + block; + + if (bat_entry->State == PAYLOAD_BLOCK_FULLY_PRESENT) + { + _u32 towrite = (std::min)(block_size - static_cast<_u32>(spos % block_size), bsize - written); + + _u32 rc = file->Write(bat_entry->FileOffsetMB * 1024 * 1024 + spos % block_size, + buffer + written, towrite); + + written += rc; + spos += rc; + + if (rc < towrite) + { + if (has_error != NULL) + *has_error = true; + + return written; + } + + continue; + } + + if (parent.get()==NULL) + { + _u32 towrite = (std::min)(block_size - static_cast<_u32>(spos % block_size), bsize - written); + + if (bat_entry->State == PAYLOAD_BLOCK_PARTIALLY_PRESENT) + { + if (has_error != NULL) + *has_error = true; + + return written; + } + else if (bat_entry->State != PAYLOAD_BLOCK_NOT_PRESENT && + bat_entry->State != PAYLOAD_BLOCK_UNDEFINED && + bat_entry->State != PAYLOAD_BLOCK_ZERO && + bat_entry->State != PAYLOAD_BLOCK_UNMAPPED) + { + if (has_error != NULL) + *has_error = true; + + return written; + } + + if (!allocateBatBlockFull(block)) + { + if (has_error != NULL) + *has_error = true; + + return written; + } + + _u32 rc = file->Write(bat_entry->FileOffsetMB * 1024 * 1024 + spos % block_size, + buffer + written, towrite); + + written += rc; + spos += rc; + + if (rc < towrite) + { + if (has_error != NULL) + *has_error = true; + + return written; + } + } + else + { + _u32 towrite = (std::min)(block_size - static_cast<_u32>(spos % block_size), bsize - written); + + if (bat_entry->State == PAYLOAD_BLOCK_PARTIALLY_PRESENT) + { + if (!setSector(spos, spos+towrite)) + { + if (has_error != NULL) + *has_error = true; + + return written; + } + } + else if (bat_entry->State != PAYLOAD_BLOCK_NOT_PRESENT && + bat_entry->State != PAYLOAD_BLOCK_UNDEFINED && + bat_entry->State != PAYLOAD_BLOCK_ZERO && + bat_entry->State != PAYLOAD_BLOCK_UNMAPPED) + { + if (has_error != NULL) + *has_error = true; + + return written; + } + else + { + if (!allocateBatBlockFull(block)) + { + if (has_error != NULL) + *has_error = true; + + return written; + } + + bat_entry->State = PAYLOAD_BLOCK_PARTIALLY_PRESENT; + + if (!setSector(spos, spos+towrite)) + { + if (has_error != NULL) + *has_error = true; + + return written; + } + } + + + _u32 rc = file->Write(bat_entry->FileOffsetMB * 1024 * 1024 + spos % block_size, + buffer + written, towrite); + + written += rc; + spos += rc; + + if (rc < towrite) + { + if (has_error != NULL) + *has_error = true; + + return written; + } + } + } + return written; +} + +_i64 VHDXFile::Size(void) +{ + return dst_size; +} + +_i64 VHDXFile::RealSize() +{ + return static_cast<_i64>(usedSize()); +} + +bool VHDXFile::PunchHole(_i64 spos, _i64 size) +{ + return false; +} + +bool VHDXFile::Sync() +{ + return syncInt(false); +} + +bool VHDXFile::syncInt(bool full) +{ + { + IScopedLock lock(pending_sector_bitmaps_mutex.get()); + + for (std::set<_u32>::iterator it = pending_sector_bitmaps.begin(); + it!=pending_sector_bitmaps.end(); ++it) + { + _u32 sector_block = *it; + + VhdxBatEntry* sector_bat_entry = reinterpret_cast(bat_buf.data()) + sector_block; + + if (sector_bat_entry->State != PAYLOAD_BLOCK_FULLY_PRESENT) + { + Server->Log("Sector bitmap bat entry not fully present when syncing", LL_WARNING); + return false; + } + + std::map<_u32, std::vector >::iterator it_sector_bitmap = sector_bitmap_bufs.find(sector_block); + if (it_sector_bitmap == sector_bitmap_bufs.end()) + { + assert(false); + return false; + } + + if (file->Write(sector_bat_entry->FileOffsetMB * 1024 * 1024, + it_sector_bitmap->second.data(), block_size) != block_size) + { + Server->Log("Error writing pending sector bitmap block. " + os_last_error_str(), LL_WARNING); + return false; + } + } + + pending_sector_bitmaps.clear(); + } + + + bool retry; + do + { + retry = false; + + IScopedLock lock(log_mutex.get()); + + int64 stop_idx = -1; + if (!fast_mode) + { + int64 new_flushed_vhdx_size = file->Size(); + if (flushed_vhdx_size != new_flushed_vhdx_size) + { + if (!file->Sync()) + { + Server->Log("Error syncing VHDX backing file -1. " + os_last_error_str(), LL_WARNING); + return false; + } + + flushed_vhdx_size = new_flushed_vhdx_size; + } + + int64 b_idx = -1; + int64 last_log_idx = -1; + for (std::set::iterator it = pending_bat_entries.begin(); + it!=pending_bat_entries.end();++it) + { + int64 entry_idx = *it; + int64 c_b_idx = (entry_idx * sizeof(int64)) / log_sector_size; + + if (b_idx != c_b_idx) + { + b_idx = c_b_idx; + + bool full = false; + if (!logWrite(bat_region.FileOffset + c_b_idx * log_sector_size, + bat_buf.data() + c_b_idx * log_sector_size, log_sector_size, -1, full)) + { + if (full) + { + stop_idx = entry_idx; + retry = true; + break; + } + else + { + Server->Log("Error logging VHDX BAT write", LL_WARNING); + return false; + } + } + } + } + + if (!file->Sync()) + { + Server->Log("Error syncing VHDX backing file -2. " + os_last_error_str(), LL_WARNING); + return false; + } + } + + check_bat_buf(); + + int64 b_idx = -1; + for (std::set::iterator it = pending_bat_entries.begin(); it != pending_bat_entries.end();) + { + int64 entry_idx = *it; + + if (entry_idx == stop_idx) + { + break; + } + + int64 c_b_idx = (entry_idx * sizeof(int64)) / log_sector_size; + + if (b_idx != c_b_idx) + { + b_idx = c_b_idx; + + _u32 rc = file->Write(bat_region.FileOffset + c_b_idx * log_sector_size, + bat_buf.data() + c_b_idx * log_sector_size, log_sector_size); + + if (rc != log_sector_size) + return false; + } + + if (stop_idx != -1) + { + std::set::iterator it_prev = it; + ++it; + pending_bat_entries.erase(it_prev); + } + else + { + ++it; + } + } + + if(stop_idx==-1) + pending_bat_entries.clear(); + + check_bat_buf(); + + if (fast_mode) + { + if (!file->Sync()) + { + Server->Log("Error syncing VHDX backing file -3. " + os_last_error_str(), LL_WARNING); + return false; + } + } + + } while (retry); + + if (full && !fast_mode) + { + if (!file->Sync()) + { + Server->Log("Error syncing VHDX backing file -4. " + os_last_error_str(), LL_WARNING); + return false; + } + + zeroGUID(curr_header.LogGuid); + + if (!updateHeader()) + return false; + } + + return true; +} + +void VHDXFile::getDataWriteGUID(VhdxGUID& g) +{ + copyGUID(curr_header.DataWriteGuid, g); +} + +bool VHDXFile::isCompressed() { + return file == compressed_file.get(); +} + +bool VHDXFile::createNew() +{ + memset(&curr_header, 0, sizeof(curr_header)); + memcpy(&curr_header, "head", 4); + curr_header.SequenceNumber = 1; + secureRandomGuid(curr_header.FileWriteGuid); + secureRandomGuid(curr_header.DataWriteGuid); + data_write_uuid_updated = true; + curr_header.Version = 1; + curr_header.LogOffset = 1 * 1024 * 1024; + curr_header.LogLength = 1 * 1024 * 1024; + curr_header.Checksum = crc32c(reinterpret_cast(&curr_header), sizeof(curr_header)); + + log_pos = 0; + log_start_pos = 0; + + block_size = 1 * 1024 * 1024; + vhdx_params.BlockSize = block_size; + sector_size = 512; + + std::vector ident = getFileIdentifier(); + + if (file->Write(0, ident.data(), static_cast<_u32>(ident.size())) != ident.size()) + { + Server->Log("Error writing new ident. " + os_last_error_str(), LL_WARNING); + return false; + } + + if (file->Write(64 * 1024, reinterpret_cast(&curr_header), sizeof(curr_header)) != sizeof(curr_header)) + { + Server->Log("Error writing new header 1. " + os_last_error_str(), LL_WARNING); + return false; + } + + if (file->Write(128 * 1024, reinterpret_cast(&curr_header), sizeof(curr_header)) != sizeof(curr_header)) + { + Server->Log("Error writing new header 2. " + os_last_error_str(), LL_WARNING); + return false; + } + + curr_header_pos = 64 * 1024; + + std::vector region_table = getVhdxRegionTable(dst_size, block_size, sector_size); + + meta_table_region.FileOffset = meta_region_offset; + meta_table_region.Length = 1 * 1024 * 1024; + + bat_region.FileOffset = bat_table_offset; + bat_region.Length = getBatLength(dst_size, block_size, sector_size); + + if (file->Write(192 * 1024, region_table.data(), static_cast<_u32>(region_table.size())) != region_table.size()) + { + Server->Log("Error writing new region table 1. " + os_last_error_str(), LL_WARNING); + return false; + } + + if (file->Write(256 * 1024, region_table.data(), static_cast<_u32>(region_table.size())) != region_table.size()) + { + Server->Log("Error writing new region table 2. " + os_last_error_str(), LL_WARNING); + return false; + } + + std::string parent_data_uuid; + std::string parent_abs_path; + std::string parent_rel_path; + if (parent.get() != NULL) + { + VhdxGUID g; + parent->getDataWriteGUID(g); + parent_data_uuid = strGUID(g); + + if (parent_fn.find("..") == 0) + { + parent_rel_path = greplace("/", "\\", parent_fn); + + std::string curr_dir = ExtractFilePath(file->getFilename()); + parent_abs_path = parent_fn; + + while (next(parent_abs_path, 0, "..\\")) + { + curr_dir = ExtractFilePath(file->getFilename()); + parent_abs_path.erase(0, 3); + } + + if (!curr_dir.empty() && curr_dir[0] == '\\') + { + curr_dir.erase(0, 1); + } + + parent_abs_path = os_file_prefix(parent_abs_path + "\\" + curr_dir); + } + else + { + parent_abs_path = os_file_prefix(parent_fn); + + std::string fn = file->getFilename(); + std::string cparent_fn = parent_fn; + + while (fn.find("\\") != std::string::npos + && cparent_fn.find("\\") != std::string::npos + && getuntil("\\", fn)==getuntil("\\", cparent_fn)) + { + fn = getafter("\\", fn); + cparent_fn = getafter("\\", cparent_fn); + } + + parent_rel_path = cparent_fn; + for (size_t i=0;i meta_region = getMetaRegion(dst_size, block_size, sector_size, + parent_data_uuid, parent_rel_path, parent_abs_path); + + if (file->Write(meta_region_offset, meta_region.data(), static_cast<_u32>(meta_region.size())) != meta_region.size()) + { + Server->Log("Error writing new metadata region. " + os_last_error_str(), LL_WARNING); + return false; + } + + if (!file->Resize(bat_region.FileOffset + bat_region.Length + allocate_size_add_size, false)) + { + Server->Log("Error writing new bat region. " + os_last_error_str(), LL_WARNING); + return false; + } + + allocated_size = file->Size(); + + next_payload_pos = bat_region.FileOffset + bat_region.Length; + + bat_buf.resize(bat_region.Length); + + return true; +} + +bool VHDXFile::updateHeader() +{ + assert(!read_only); + + ++curr_header.SequenceNumber; + + curr_header.Checksum = 0; + curr_header.Checksum = crc32c(reinterpret_cast(&curr_header), sizeof(curr_header)); + + if (file->Write(curr_header_pos, reinterpret_cast(&curr_header), sizeof(curr_header)) != sizeof(curr_header)) + { + Server->Log("Error writing VHDX header to pos " + convert(curr_header_pos) + ". " + os_last_error_str()); + return false; + } + + if (!file->Sync()) + { + Server->Log("Error syncing VHDX backing file after updating header. " + os_last_error_str(), LL_WARNING); + return false; + } + + if (curr_header_pos == 64 * 1024) + curr_header_pos = 128 * 1024; + else + curr_header_pos = 64 * 1024; + + return true; +} + +bool VHDXFile::replayLog() +{ + VHDXFile::LogSequence seq = findLogSequence(); + + if (seq.max_sequence == 0) + { + Server->Log("Could not find VHDX log sequence -1", LL_WARNING); + return false; + } + + if (seq.entries.empty()) + { + Server->Log("Could not find VHDX log sequence -2", LL_WARNING); + return false; + } + + LogEntry head_entry = readLogEntry(file, curr_header.LogGuid, seq.entries[seq.entries.size()-1]); + + if (file->Size() < head_entry.fsize) + { + Server->Log("VHDX size smaller than expected from log expected="+convert(head_entry.fsize) + +" got="+convert(file->Size()), LL_WARNING); + return false; + } + + for (size_t i=0;iLog("Error reading log entry while replaying log", LL_WARNING); + return false; + } + + if (file->Size() < loge.fsize) + { + Server->Log("VHDX size smaller than expected from log entry expected=" + convert(loge.fsize) + + " got=" + convert(file->Size()), LL_WARNING); + return false; + } + + for (size_t j=0;j zero_buf(zero_desc.ZeroLength); + if (file->Write(zero_desc.FileOffset, zero_buf.data(), static_cast<_u32>(zero_buf.size())) != zero_buf.size()) + { + Server->Log("Error writing zeroes from log. " + os_last_error_str(), LL_WARNING); + return false; + } + } + + for (size_t j=0;jWrite(log_data.offset, log_data.data, sizeof(log_data.data)) != sizeof(log_data.data)) + { + Server->Log("Error writing data from log. " + os_last_error_str(), LL_WARNING); + return false; + } + } + + log_sequence_num = loge.sequence_number + 1; + } + + int64 new_fsize = -1; + if (file->Size() < head_entry.new_fsize) + { + if (file->Resize(head_entry.new_fsize, false)) + new_fsize = head_entry.new_fsize; + } + + if (!file->Sync()) + { + Server->Log("Error syncing after writing log. " + os_last_error_str()); + return false; + } + + if (new_fsize >= 0) + flushed_vhdx_size = new_fsize; + + zeroGUID(curr_header.LogGuid); + + return updateHeader(); +} + +bool VHDXFile::readHeader() +{ + std::string ident = file->Read(0LL, 8); + + if (ident != "vhdxfile") + { + Server->Log("VHDX header tag wrong", LL_WARNING); + return false; + } + + VhdxHeader header1, header2; + + if (file->Read(64LL * 1024, reinterpret_cast(&header1), sizeof(header1)) != sizeof(header1)) + { + Server->Log("Could not read VHDX header 1. " + os_last_error_str(), LL_WARNING); + return false; + } + + if (file->Read(128LL * 1024, reinterpret_cast(&header2), sizeof(header2)) != sizeof(header2)) + { + Server->Log("Could not read VHDX header 2. " + os_last_error_str(), LL_WARNING); + return false; + } + + VhdxHeader* sel_header = NULL; + + if (checkHeader(file, header1)) + { + sel_header = &header1; + } + + if (checkHeader(file, header2) && + header2.SequenceNumber > header1.SequenceNumber) + { + sel_header = &header2; + } + + if (sel_header == NULL) + { + Server->Log("Both VHDX headers are invalid", LL_WARNING); + return false; + } + + memcpy(&curr_header, sel_header, sizeof(curr_header)); + + return true; +} + +bool VHDXFile::readRegionTable(int64 off) +{ + std::vector region_buf(64 * 1024); + + if (file->Read(off, region_buf.data(), static_cast<_u32>(region_buf.size())) != region_buf.size()) + { + Server->Log("Error reading VHDX region table. " + os_last_error_str(), LL_WARNING); + return false; + } + + if (std::string(region_buf.data(), 4) != "regi") + { + Server->Log("VHDX region table tag wrong", LL_WARNING); + return false; + } + + VhdxRegionTableHeader* header = reinterpret_cast(region_buf.data()); + + _u32 ccrc = header->Checksum; + + header->Checksum = 0; + + if (crc32c(reinterpret_cast(region_buf.data()), region_buf.size()) != ccrc) + { + Server->Log("VHDX region table checksum wrong", LL_WARNING); + return false; + } + + VhdxGUID meta_table_guid; + makeMetaTableGUID(meta_table_guid); + VhdxGUID bat_guid; + makeBatGUID(bat_guid); + + unsigned int found = 0; + + for (_u32 i = 0; i < header->EntryCount; ++i) + { + VhdxRegionTableEntry* entry = reinterpret_cast(region_buf.data() + sizeof(VhdxRegionTableHeader) + + i*sizeof(VhdxRegionTableEntry)); + + if (equalsGUID(entry->Guid, meta_table_guid)) + { + memcpy(&meta_table_region, entry, sizeof(meta_table_region)); + if (found & 1) + { + Server->Log("Found metadata table region entry twice", LL_WARNING); + return false; + } + found |= 1; + } + else if (equalsGUID(entry->Guid, bat_guid)) + { + memcpy(&bat_region, entry, sizeof(bat_region)); + if (found & 2) + { + Server->Log("Found BAT table region entry twice", LL_WARNING); + return false; + } + found |= 2; + } + else + { + Server->Log("Unknown region table entry " + strGUID(entry->Guid), LL_WARNING); + return false; + } + } + + if ((found ^ (1 | 2)) != 0) + { + Server->Log("Did not find required region table entry. Found="+convert(found), LL_WARNING); + return false; + } + + return true; +} + +bool VHDXFile::readBat() +{ + bat_buf.resize(bat_region.Length); + + const _u32 read_size = 512 * 1024; + + for (_u32 i = 0; i < bat_region.Length; i += read_size) + { + _u32 toread = (std::min)(read_size, bat_region.Length - i); + if (file->Read(bat_region.FileOffset + i, bat_buf.data() + i, toread) != toread) + { + Server->Log("Error reading VHDX BAT at pos " + convert(bat_region.FileOffset + i) + + " toread " + convert(toread) + ". " + os_last_error_str(), LL_WARNING); + return false; + } + } + + check_bat_buf(); + + return true; +} + +bool VHDXFile::readMeta() +{ + std::vector meta_table(64 * 1024); + + if (meta_table_region.Length < 64 * 1024) + { + Server->Log("Meta table region length smaller than 64KiB", LL_WARNING); + return false; + } + + if (file->Read(meta_table_region.FileOffset, meta_table.data(), static_cast<_u32>(meta_table.size())) != meta_table.size()) + { + Server->Log("Error reading VHDX meta table from pos " + + convert(meta_table_region.FileOffset) + ". " + os_last_error_str(), LL_WARNING); + return false; + } + + VhdxMetadataTableHeader* table_header = reinterpret_cast(meta_table.data()); + + std::string ident(meta_table.data(), 8); + + if (ident != "metadata") + { + Server->Log("Meta table ident wrong", LL_WARNING); + return false; + } + + sector_size = 0; + physical_sector_size = 0; + vhdx_params.BlockSize = 0; + dst_size = -1; + + VhdxGUID parent_linkage_guid = {}; + VhdxGUID file_parameters_guid, virtual_disk_size_guid, logical_sector_size_guid, + physical_sector_size_guid, virtual_disk_id_guid, parent_locator_guid; + + makeFileParametersGUID(file_parameters_guid); + makeVirtualDiskSizeGUID(virtual_disk_size_guid); + makeLogicalSectorSizeGUID(logical_sector_size_guid); + makePhysicalSectorSizeGUID(physical_sector_size_guid); + makeVirtualDiskIdGUID(virtual_disk_id_guid); + makeParentLocatorGUID(parent_locator_guid); + + std::string rel_parent_path; + std::string volume_parent_path; + std::string absolute_win32_parent_path; + + for (unsigned short i = 0; i < table_header->EntryCount; ++i) + { + if (32 + i * 32 + 32 > meta_table.size()) + { + Server->Log("Meta table not large enough", LL_WARNING); + return false; + } + + VhdxMetadataTableEntry* table_entry = reinterpret_cast(meta_table.data() + 32 + i * 32); + + if (table_entry->Offset < 64 * 1024) + { + Server->Log("Meta table offset wrong: " + convert(table_entry->Offset), LL_WARNING); + return false; + } + if (table_entry->Offset + table_entry->Length > meta_table_region.Length) + { + Server->Log("Meta table offset+length wrong: " + convert(table_entry->Offset + table_entry->Length), LL_WARNING); + return false; + } + + std::vector entry_buf(table_entry->Length); + + if (file->Read(meta_table_region.FileOffset + table_entry->Offset, + entry_buf.data(), static_cast<_u32>(entry_buf.size())) != entry_buf.size()) + { + Server->Log("Error reading meta table entry. " + os_last_error_str(), LL_WARNING); + return false; + } + + if (equalsGUID(table_entry->ItemId, file_parameters_guid)) + { + if (entry_buf.size() < sizeof(VhdxFileParameters)) + { + Server->Log("VhdxFileParameters entry not large enough", LL_WARNING); + return false; + } + + memcpy(&vhdx_params, entry_buf.data(), sizeof(vhdx_params)); + } + else if (equalsGUID(table_entry->ItemId, virtual_disk_size_guid)) + { + if (entry_buf.size() < sizeof(VhdxVirtualDiskSize)) + { + Server->Log("VhdxVirtualDiskSize entry not large enough", LL_WARNING); + return false; + } + + VhdxVirtualDiskSize* virtual_disk_size = reinterpret_cast(entry_buf.data()); + + dst_size = virtual_disk_size->VirtualDiskSize; + } + else if (equalsGUID(table_entry->ItemId, physical_sector_size_guid)) + { + if (entry_buf.size() < sizeof(VhdxPhysicalDiskSectorSize)) + { + Server->Log("VhdxPhysicalDiskSectorSize entry not large enough", LL_WARNING); + return false; + } + + VhdxPhysicalDiskSectorSize* physical_disk_sector_size = reinterpret_cast(entry_buf.data()); + + physical_sector_size = physical_disk_sector_size->PhysicalSectorSize; + } + else if (equalsGUID(table_entry->ItemId, logical_sector_size_guid)) + { + if (entry_buf.size() < sizeof(VhdxVirtualDiskLogicalSectorSize)) + { + Server->Log("VhdxVirtualDiskLogicalSectorSize entry not large enough", LL_WARNING); + return false; + } + + VhdxVirtualDiskLogicalSectorSize* logical_disk_sector_size = reinterpret_cast(entry_buf.data()); + + sector_size = logical_disk_sector_size->LogicalSectorSize; + } + else if (equalsGUID(table_entry->ItemId, virtual_disk_id_guid)) + { + if (entry_buf.size() < sizeof(VhdxVirtualDiskId)) + { + Server->Log("VhdxVirtualDiskId entry not large enough", LL_WARNING); + return false; + } + + VhdxVirtualDiskId* virtual_disk_id = reinterpret_cast(entry_buf.data()); + } + else if (equalsGUID(table_entry->ItemId, parent_locator_guid)) + { + if (entry_buf.size() < sizeof(VhdxParentLocatorHeader)) + { + Server->Log("Parent locator entry not large enough", LL_WARNING); + return false; + } + + VhdxParentLocatorHeader* parent_locator_header = reinterpret_cast(entry_buf.data()); + + VhdxGUID vhdx_parent_locator_guid; + makeVhdxParentLocatorGUID(vhdx_parent_locator_guid); + + if (!equalsGUID(parent_locator_header->LocatorType, vhdx_parent_locator_guid)) + { + Server->Log("Unknown parent locator type " + strGUID(parent_locator_header->LocatorType), LL_WARNING); + return false; + } + + for (unsigned short i = 0; i < parent_locator_header->KeyValueCount; ++i) + { + VhdxParentLocatorEntry* parent_locator_entry = reinterpret_cast(entry_buf.data() + 20 + i * 12); + + if (parent_locator_entry->KeyOffset + parent_locator_entry->KeyLength > entry_buf.size() + || parent_locator_entry->KeyOffset>10*1024*1024) + { + Server->Log("Parent locator entry key offset not plausible: "+convert(parent_locator_entry->KeyOffset)+ + " length: "+convert(parent_locator_entry->KeyLength)+" entry_buf.size()="+convert(entry_buf.size()), + LL_WARNING); + return false; + } + + if (parent_locator_entry->ValueOffset + parent_locator_entry->ValueLength > entry_buf.size() + || parent_locator_entry->ValueOffset > 10 * 1024 * 1024) + { + Server->Log("Parent locator entry value offset not plausible: " + convert(parent_locator_entry->ValueOffset)+ + " length: " + convert(parent_locator_entry->ValueLength) + " entry_buf.size()=" + convert(entry_buf.size()), + LL_WARNING); + return false; + } + + std::string key_vw(entry_buf.data() + parent_locator_entry->KeyOffset, parent_locator_entry->KeyLength); + std::string value_vw(entry_buf.data() + parent_locator_entry->ValueOffset, parent_locator_entry->ValueLength); + + std::string key_v = Server->ConvertFromUTF16(key_vw); + std::string value_v = Server->ConvertFromUTF16(value_vw); + + if (key_v == "parent_linkage") + { + if (!parseStrGuid(value_v, parent_linkage_guid)) + { + Server->Log("Error parsing parent linkage GUID " + value_v, LL_WARNING); + return false; + } + } + else if (key_v == "relative_path") + { + rel_parent_path = value_v; + } + else if (key_v == "volume_path") + { + volume_parent_path = value_v; + } + else if (key_v == "absolute_win32_path") + { + absolute_win32_parent_path = value_v; + } + } + } + else if(table_entry->IsRequired) + { + Server->Log("Required table entry " + strGUID(table_entry->ItemId) + " not suppoerted", LL_WARNING); + return false; + } + } + + if (sector_size == 0 || + physical_sector_size == 0 || + vhdx_params.BlockSize == 0 || + dst_size == -1) + { + Server->Log("Missing VHDX parameter. sector_size=" + convert(sector_size) + + " physical_sector_size=" + convert(physical_sector_size) + + " vhdx_params.BlockSize=" + convert(vhdx_params.BlockSize)+ + " dst_size=" + convert(dst_size), LL_WARNING); + return false; + } + + block_size = vhdx_params.BlockSize; + + if (vhdx_params.HasParent) + { + if (isZeroGUID(parent_linkage_guid)) + { + Server->Log("Parent linkage GUID is zero", LL_WARNING); + return false; + } + + if (FileExists(absolute_win32_parent_path)) + { + parent.reset(new VHDXFile(absolute_win32_parent_path, + true, 0)); + } + else if (FileExists(rel_parent_path)) + { + parent.reset(new VHDXFile(rel_parent_path, + true, 0)); + } + + if (parent.get() == NULL || + !parent->isOpen()) + { + Server->Log("Could not open parent vhdx at \"" + absolute_win32_parent_path + "\" or " + "\"" + rel_parent_path + "\"", LL_WARNING); + return false; + } + + VhdxGUID dwg; + parent->getDataWriteGUID(dwg); + + if (!equalsGUID(dwg, parent_linkage_guid)) + { + Server->Log("Parent linkage GUID differs. Got " + strGUID(dwg) + " expected " + strGUID(parent_linkage_guid), LL_WARNING); + return false; + } + } + + return true; +} + +bool VHDXFile::allocateBatBlockFull(int64 block) +{ + VhdxBatEntry* bat_entry = reinterpret_cast(bat_buf.data()) + block; + + bat_entry->State = PAYLOAD_BLOCK_FULLY_PRESENT; + + int64 new_pos; + { + IScopedLock lock(next_payload_mutex.get()); + new_pos = next_payload_pos; + next_payload_pos += block_size; + } + + if (new_pos > file->Size()) + { + allocated_size = new_pos + block_size + allocate_size_add_size; + + if (!file->Resize(allocated_size, false)) + { + Server->Log("Error resizing backing file to new allocated size " + + convert(allocated_size) + ". " + os_last_error_str(), + LL_WARNING); + return false; + } + } + + assert(new_pos % (1 * 1024 * 1024) == 0); + bat_entry->FileOffsetMB = new_pos / (1 * 1024 * 1024); + bat_entry->Reserved = 0; + + { + IScopedLock lock(log_mutex.get()); + pending_bat_entries.insert(block); + } + + return true; +} + +void VHDXFile::calcNextPayloadPos() +{ + int64 next_pos = 1 * 1024 * 1024; + + next_pos = (std::max)(next_pos, + static_cast(bat_region.FileOffset + bat_region.Length)); + + next_pos = (std::max)(next_pos, + static_cast(curr_header.LogOffset + curr_header.LogLength)); + + next_pos = (std::max)(next_pos, + static_cast(meta_table_region.FileOffset + meta_table_region.Length)); + + _u32 bat_entries = getBatEntries(dst_size, block_size, sector_size); + + for (_u32 i = 0; i < bat_entries; ++i) + { + VhdxBatEntry* bat_entry = reinterpret_cast(bat_buf.data()) + i; + next_pos = (std::max)(next_pos, + static_cast(bat_entry->FileOffsetMB*1024*1024 + block_size)); + } + + next_payload_pos = next_pos; +} + +bool VHDXFile::open(const std::string& fn, bool compress, size_t compress_n_threads) +{ + backing_file_holder.reset(Server->openFile(fn, read_only ? MODE_READ : MODE_RW_CREATE)); + + if (backing_file_holder.get() == NULL) + { + Server->Log("Error opening VHDX backing file at \"" + + fn + "\". " + os_last_error_str(), LL_WARNING); + return false; + } + + backing_file = backing_file_holder.get(); + + if (backing_file->Size() == 0) + { + if (read_only) + { + Server->Log("Read only vhdx file has zero size", LL_WARNING); + return false; + } + + if (compress) + { + compressed_file.reset(new CompressedFile(backing_file, + false, read_only, + compress_n_threads == 0 ? VHDFile::getNumCompThreads(read_only) : compress_n_threads)); + + backing_file_holder.release(); + + if (compressed_file->hasError()) + { + Server->Log("Error opening VHDX compressed file -1", LL_WARNING); + return false; + } + + file = compressed_file.get(); + } + else + { + file = backing_file; + } + + return createNew(); + } + else + { + if (check_if_compressed()) + { + compressed_file.reset(new CompressedFile(backing_file, + true, read_only, + compress_n_threads == 0 ? VHDFile::getNumCompThreads(read_only) : compress_n_threads)); + + backing_file_holder.release(); + + if (compressed_file->hasError()) + { + Server->Log("Error opening VHDX compressed file -2", LL_WARNING); + return false; + } + + file = compressed_file.get(); + } + else + { + file = backing_file; + } + + if (!readHeader()) + { + Server->Log("Error reading VHDX header", LL_WARNING); + return false; + } + + if (!readRegionTable(192 * 1024) && + !readRegionTable(256 * 1024)) + { + Server->Log("Error reading any VHDX region table", LL_WARNING); + return false; + } + + if (!readBat()) + { + Server->Log("Error reading any VHDX bat", LL_WARNING); + return false; + } + + if (!readMeta()) + { + Server->Log("Error reading any VHDX metadata", LL_WARNING); + return false; + } + + if (read_only && !isZeroGUID(curr_header.LogGuid)) + { + Server->Log("VHDX is opened read only but has log entries", LL_WARNING); + return false; + } + + if (!read_only && !isZeroGUID(curr_header.LogGuid)) + { + if (!replayLog()) + { + Server->Log("Error replaying VHDX log", LL_WARNING); + return false; + } + } + + calcNextPayloadPos(); + + allocated_size = backing_file->Size(); + + if(!read_only) + secureRandomGuid(curr_header.FileWriteGuid); + + flushed_vhdx_size = allocated_size; + + if (!read_only && !fast_mode && !updateHeader()) + { + return false; + } + + return true; + } +} + +bool VHDXFile::has_sector_int(int64 spos) +{ + if (spos >= dst_size) + return true; + + _u32 block = getBatEntry(spos, block_size, sector_size); + + VhdxBatEntry* bat_entry = reinterpret_cast(bat_buf.data()) + block; + + return bat_entry->State == PAYLOAD_BLOCK_FULLY_PRESENT || + bat_entry->State == PAYLOAD_BLOCK_PARTIALLY_PRESENT; +} + +VHDXFile::LogSequence VHDXFile::findLogSequence() +{ + LogSequence max_seq; + max_seq.max_sequence = 0; + for (uint64 log_pos = curr_header.LogOffset; + log_pos < curr_header.LogOffset + curr_header.LogLength;) + { + LogSequence seq = findLogSequence(log_pos); + if (seq.max_sequence > max_seq.max_sequence) + max_seq = seq; + } + + return max_seq; +} + +VHDXFile::LogSequence& VHDXFile::validateSequence(LogSequence& seq) +{ + if (seq.entries.empty()) + return seq; + + LogEntry head = readLogEntry(file, curr_header.LogGuid, + static_cast(seq.entries[seq.entries.size() - 1])); + + if (head.sequence_number == -1) + { + seq.entries.clear(); + return seq; + } + + if (curr_header.LogOffset + head.tail_pos != seq.entries[0]) + { + seq.entries.clear(); + return seq; + } + + return seq; +} + +VHDXFile::LogSequence VHDXFile::findLogSequence(uint64& off) +{ + int64 expected_seq = 0; + VHDXFile::LogSequence seq; + + while (true) + { + LogEntry loge = readLogEntry(file, curr_header.LogGuid, static_cast(off)); + + if (loge.sequence_number == -1) + { + off += 4096; + return validateSequence(seq); + } + + if (expected_seq != 0 && expected_seq != loge.sequence_number) + { + return validateSequence(seq); + } + + seq.entries.push_back(off); + seq.max_sequence = loge.sequence_number; + seq.fsize = loge.fsize; + + off += loge.length; + off = (off - curr_header.LogOffset) % curr_header.LogLength + curr_header.LogOffset; + + expected_seq = loge.sequence_number + 1; + } +} + +bool VHDXFile::logWrite(int64 off, const char* buf, size_t bsize, + int64 new_dst_size, bool& full) +{ + if (bsize > 126 * log_sector_size) + { + assert(false); + return false; + } + + assert(bsize % log_sector_size == 0); + + if (isZeroGUID(curr_header.LogGuid)) + { + randomGuid(curr_header.LogGuid); + log_pos = 0; + log_start_pos = 0; + + if (!updateHeader()) + return false; + } + + size_t desc_count = roundUp(bsize, static_cast(log_sector_size)) / log_sector_size; + + std::vector log_entry(log_sector_size + roundUp(bsize, static_cast(log_sector_size) ) ); + + if (log_pos + log_entry.size() > curr_header.LogLength) + { + full = true; + return false; + } + + LogEntryHeader* header = reinterpret_cast(log_entry.data()); + + memcpy(&header->signature, "loge", 4); + + header->Checksum = 0; + header->EntryLength = static_cast<_u32>(log_entry.size()); + header->DescriptorCount = static_cast<_u32>(desc_count); + header->Tail = static_cast<_u32>(log_start_pos); + header->FlushedFileOffset = flushed_vhdx_size; + if (new_dst_size <= 0) + header->LastFileOffset = header->FlushedFileOffset; + else + header->LastFileOffset = new_dst_size; + copyGUID(curr_header.LogGuid, header->LogGuid); + header->SequenceNumber = log_sequence_num; + + ++log_sequence_num; + + for (size_t i = 0; i < desc_count; ++i) + { + LogDataDescriptor* data_desc = reinterpret_cast(log_entry.data() + 64 + i * 32); + + memcpy(&data_desc->signature, "desc", 4); + data_desc->FileOffset = off + i * log_sector_size; + memcpy(data_desc->LeadingBytes, buf + i * log_sector_size, 8); + memcpy(data_desc->TrailingBytes, buf + i * log_sector_size + (log_sector_size - 4), 4); + data_desc->SequenceNumber = header->SequenceNumber; + } + + for (size_t i = 0; i < bsize; i += log_sector_size) + { + LogDataSector* data_sec = reinterpret_cast(log_entry.data() + log_sector_size + i * log_sector_size); + + memcpy(&data_sec->signature, "data", 4); + SSequence seq; + seq.QuadPart = header->SequenceNumber; + data_sec->SequenceLow = seq.LowPart; + data_sec->SequenceHigh = seq.HighPart; + memcpy(data_sec->data, buf + i + 8, log_sector_size - 8 - 4); + } + + header->Checksum = crc32c(reinterpret_cast(log_entry.data()), log_entry.size()); + + if (file->Write(curr_header.LogOffset + log_pos, log_entry.data(), + static_cast<_u32>(log_entry.size())) != log_entry.size()) + { + Server->Log("Error writing VHDX log entry. " + os_last_error_str(), LL_WARNING); + return false; + } + + log_pos += log_entry.size(); + + return true; +} + +char* VHDXFile::getSectorBitmap(_u32 sector_block, uint64 FileOffsetMB) +{ + IScopedLock lock(sector_bitmap_mutex.get()); + std::map<_u32, std::vector >::iterator it_sector_bitmap = sector_bitmap_bufs.find(sector_block); + if (it_sector_bitmap == sector_bitmap_bufs.end()) + { + lock.relock(NULL); + + std::vector sector_bitmap_buf(block_size); + + if (file->Read(FileOffsetMB * 1024 * 1024, + sector_bitmap_buf.data(), + static_cast<_u32>(sector_bitmap_buf.size())) != block_size) + { + Server->Log("Reading sector bitmap from mb offset " + convert(FileOffsetMB) + + " failed. " + os_last_error_str(), LL_ERROR); + return NULL; + } + + lock.relock(sector_bitmap_mutex.get()); + + if (sector_bitmap_bufs.find(sector_block) == sector_bitmap_bufs.end()) + { + sector_bitmap_bufs[sector_block] = sector_bitmap_buf; + } + + it_sector_bitmap = sector_bitmap_bufs.find(sector_block); + + lock.relock(NULL); + } + + return it_sector_bitmap->second.data(); +} + +char* VHDXFile::addZeroBitmap(_u32 sector_block) +{ + IScopedLock lock(sector_bitmap_mutex.get()); + std::map<_u32, std::vector >::iterator it_sector_bitmap = sector_bitmap_bufs.find(sector_block); + if (it_sector_bitmap == sector_bitmap_bufs.end()) + { + std::vector sector_bitmap_buf(block_size); + return sector_bitmap_bufs.insert(std::make_pair(sector_block, sector_bitmap_buf)).first->second.data(); + } + + return it_sector_bitmap->second.data(); +} + +bool VHDXFile::isSectorSet(int64 spos, bool& set) +{ + _u32 sector_block = getSectorBitmapEntry(spos, block_size, sector_size); + VhdxBatEntry* sector_bat_entry = reinterpret_cast(bat_buf.data()) + sector_block; + + if (sector_bat_entry->State != PAYLOAD_BLOCK_FULLY_PRESENT) + { + Server->Log("Sector bitmap " + convert(sector_block) + " not fully present", LL_WARNING); + return false; + } + + char* sector_bitmap = getSectorBitmap(sector_block, sector_bat_entry->FileOffsetMB); + + if (sector_bitmap == NULL) + { + Server->Log("Error reading sector bitmap of sector block " + convert(sector_block), LL_ERROR); + return false; + } + + set = isSectorSetInt(sector_bitmap, spos, block_size, sector_size); + + return true; +} + +bool VHDXFile::setSector(int64 spos) +{ + return setSector(spos, spos + sector_size); +} + +bool VHDXFile::setSector(int64 start, int64 end) +{ + _u32 sector_block = getSectorBitmapEntry(start, block_size, sector_size); + + VhdxBatEntry* sector_bat_entry = reinterpret_cast(bat_buf.data()) + sector_block; + + if (sector_bat_entry->State != PAYLOAD_BLOCK_FULLY_PRESENT && + sector_bat_entry->State != PAYLOAD_BLOCK_NOT_PRESENT) + { + Server->Log("Sector bitmap " + convert(sector_block) + " wrong state " + +convert(sector_bat_entry->State), LL_WARNING); + return false; + } + + char* sector_bitmap; + if (sector_bat_entry->State == PAYLOAD_BLOCK_NOT_PRESENT) + { + if (!allocateBatBlockFull(sector_block)) + return false; + + sector_bitmap = addZeroBitmap(sector_block); + } + else + { + sector_bitmap = getSectorBitmap(sector_block, sector_bat_entry->FileOffsetMB); + } + + if (sector_bitmap == NULL) + return false; + + setSectorInt(sector_bitmap, start, end, block_size, sector_size); + + IScopedLock lock(pending_sector_bitmaps_mutex.get()); + + pending_sector_bitmaps.insert(sector_block); + + return true; +} + +bool VHDXFile::check_if_compressed() +{ + const char header_magic[] = "URBACKUP COMPRESSED FILE"; + std::string magic = backing_file->Read(0LL, sizeof(header_magic) - 1); + + return magic == std::string(header_magic); +} + +bool VHDXFile::has_block(bool use_parent) +{ + if (!has_sector_int(spos)) + { + if (use_parent && parent.get() != NULL) + return parent->has_block(true); + + return false; + } + + return true; +} + +void VHDXFile::check_bat_buf() +{ +#ifndef NDEBUG + for (size_t i = 0; i < bat_buf.size(); i += sizeof(VhdxBatEntry)) + { + const VhdxBatEntry* entry = reinterpret_cast(bat_buf.data() + i); + assert(entry->State != 5); + assert(entry->State != 4); + } +#endif +} diff --git a/fsimageplugin/vhdxfile.h b/fsimageplugin/vhdxfile.h new file mode 100644 index 000000000..9ff69d796 --- /dev/null +++ b/fsimageplugin/vhdxfile.h @@ -0,0 +1,195 @@ +#pragma once + +#include "../Interface/Server.h" +#include "../Interface/File.h" +#include "IVHDFile.h" + +#include +#include + +class CompressedFile; + +typedef char VhdxGUID[16]; + +#pragma pack(1) +struct VhdxHeader +{ + _u32 Signature; + _u32 Checksum; + uint64 SequenceNumber; + VhdxGUID FileWriteGuid; + VhdxGUID DataWriteGuid; + VhdxGUID LogGuid; + unsigned short LogVersion; + unsigned short Version; + _u32 LogLength; + uint64 LogOffset; + char Reserved[4016]; +}; + +struct VhdxRegionTableEntry +{ + VhdxGUID Guid; + uint64 FileOffset; + _u32 Length; + _u32 Required : 1; + _u32 Reserved : 31; +}; + +struct VhdxFileParameters +{ + _u32 BlockSize; + _u32 LeaveBlocksAllocated : 1; + _u32 HasParent : 1; + _u32 Reserved : 30; +}; + +struct VhdxBatEntry +{ + uint64 State : 3; + uint64 Reserved : 17; + uint64 FileOffsetMB : 44; +}; +#pragma pack() + +class VHDXFile : public IVHDFile, public IFile +{ +public: + + VHDXFile(const std::string& fn, bool pRead_only, uint64 pDstsize, unsigned int pBlocksize = 2 * 1024 * 1024, bool fast_mode = false, bool compress = false, size_t compress_n_threads = 0); + VHDXFile(const std::string& fn, const std::string& parent_fn, bool pRead_only, bool fast_mode = false, bool compress = false, uint64 pDstsize = 0, size_t compress_n_threads = 0); + ~VHDXFile(); + + virtual bool Seek(_i64 offset); + virtual bool Read(char* buffer, size_t bsize, size_t& read); + virtual _u32 Write(const char* buffer, _u32 bsize, bool* has_error = NULL); + virtual bool isOpen(void); + virtual uint64 getSize(void); + virtual uint64 usedSize(void); + virtual std::string getFilename(void); + virtual bool has_sector(_i64 sector_size = -1); + virtual bool this_has_sector(_i64 sector_size = -1); + virtual unsigned int getBlocksize(); + virtual bool finish(); + virtual bool trimUnused(_i64 fs_offset, _i64 trim_blocksize, ITrimCallback* trim_callback); + virtual bool syncBitmap(_i64 fs_offset); + virtual bool makeFull(_i64 fs_offset, IVHDWriteCallback* write_callback); + virtual bool setUnused(_i64 unused_start, _i64 unused_end); + virtual bool setBackingFileSize(_i64 fsize); + + virtual std::string Read(_u32 tr, bool* has_error = NULL); + virtual std::string Read(int64 spos, _u32 tr, bool* has_error = NULL); + virtual _u32 Read(char* buffer, _u32 bsize, bool* has_error = NULL); + virtual _u32 Read(int64 spos, char* buffer, _u32 bsize, bool* has_error = NULL); + virtual _u32 Write(const std::string& tw, bool* has_error = NULL); + virtual _u32 Write(int64 spos, const std::string& tw, bool* has_error = NULL); + virtual _u32 Write(int64 spos, const char* buffer, _u32 bsiz, bool* has_error = NULL); + virtual _i64 Size(void); + virtual _i64 RealSize(); + virtual bool PunchHole(_i64 spos, _i64 size); + virtual bool Sync(); + + void getDataWriteGUID(VhdxGUID& g); + + VHDXFile* getParent() { + return parent.get(); + } + + bool isCompressed(); + +private: + bool createNew(); + bool updateHeader(); + bool replayLog(); + bool readHeader(); + bool readRegionTable(int64 off); + bool readBat(); + bool readMeta(); + bool allocateBatBlockFull(int64 block); + void calcNextPayloadPos(); + bool open(const std::string& fn, bool compress, size_t compress_n_threads); + bool syncInt(bool full); + + bool has_sector_int(int64 spos); + + struct LogSequence + { + LogSequence() + : max_sequence(-1), + fsize(-1), + new_fsize(-1) {} + + std::vector entries; + int64 max_sequence; + int64 fsize; + int64 new_fsize; + }; + + LogSequence findLogSequence(); + LogSequence& validateSequence(LogSequence& seq); + LogSequence findLogSequence(uint64& off); + + bool logWrite(int64 off, const char* buf, size_t bsize, int64 new_dst_size, bool& full); + + char* getSectorBitmap(_u32 sector_block, uint64 FileOffsetMB); + char* addZeroBitmap(_u32 sector_block); + + bool isSectorSet(int64 spos, bool& set); + bool setSector(int64 spos); + bool setSector(int64 start, int64 end); + + bool check_if_compressed(); + + bool has_block(bool use_parent); + + void check_bat_buf(); + + VhdxHeader curr_header; + int64 curr_header_pos; + + VhdxRegionTableEntry meta_table_region; + VhdxRegionTableEntry bat_region; + + VhdxFileParameters vhdx_params; + + std::vector bat_buf; + std::set pending_bat_entries; + + IFsFile* backing_file; + std::auto_ptr backing_file_holder; + IFsFile* file; + std::auto_ptr compressed_file; + int64 allocated_size; + bool is_open; + int64 dst_size; + int64 flushed_vhdx_size; + bool data_write_uuid_updated; + + std::auto_ptr log_mutex; + + _u32 sector_size; + _u32 physical_sector_size; + _u32 block_size; + + std::auto_ptr next_payload_mutex; + int64 next_payload_pos; + + int64 log_start_pos; + int64 log_pos; + int64 log_sequence_num; + + int64 spos; + + bool fast_mode; + bool read_only; + bool finished; + + std::auto_ptr parent; + std::string parent_fn; + + std::auto_ptr sector_bitmap_mutex; + std::map<_u32, std::vector > sector_bitmap_bufs; + + std::auto_ptr pending_sector_bitmaps_mutex; + std::set<_u32> pending_sector_bitmaps; +}; diff --git a/fuseplugin/dllmain.cpp b/fuseplugin/dllmain.cpp index 5f9082da1..153adb615 100644 --- a/fuseplugin/dllmain.cpp +++ b/fuseplugin/dllmain.cpp @@ -164,8 +164,22 @@ DLLEXPORT void LoadActions(IServer* pServer) Server->Log("Error loading fsimageplugin", LL_ERROR); exit(2); } - - vhdfile = image_fak->createVHDFile(vhd_filename, true, 0); + + std::string extension = findextension(vhd_filename); + + if (extension == "vhdx" + || extension == "vhdxz") + { + vhdfile = image_fak->createVHDFile(vhd_filename, true, 0, 2 * 1024 * 1024, false, IFSImageFactory::ImageFormat_VHDX); + } + else if(extension=="raw") + { + vhdfile = image_fak->createVHDFile(vhd_filename, true, 0, 2 * 1024 * 1024, false, IFSImageFactory::ImageFormat_RawCowFile); + } + else + { + vhdfile = image_fak->createVHDFile(vhd_filename, true, 0); + } if(vhdfile==NULL || !vhdfile->isOpen()) { diff --git a/httpserver/HTTPClient.cpp b/httpserver/HTTPClient.cpp index 6093da25a..2d45f45b1 100644 --- a/httpserver/HTTPClient.cpp +++ b/httpserver/HTTPClient.cpp @@ -528,9 +528,10 @@ bool CHTTPClient::processRequest(void) size_t pstart; str_map::iterator upgrade_param = http_params.find("UPGRADE"); if (upgrade_param != http_params.end() - && upgrade_param->second == "websocket") + && strlower(upgrade_param->second) == "websocket") { std::string name = getuntil("?", *pl); + if (name.empty()) name = *pl; std::string gparams = getafter("?", *pl); CHTTPSocket* socket_handler = new CHTTPSocket(name, gparams, http_params, pipe, endpoint); request_ticket = Server->getThreadPool()->execute(socket_handler, "http websocket"); diff --git a/httpserver/httpserver.vcxproj b/httpserver/httpserver.vcxproj index c7015bc9b..4799634df 100644 --- a/httpserver/httpserver.vcxproj +++ b/httpserver/httpserver.vcxproj @@ -1,6 +1,10 @@  + + Debug + ARM64 + Debug Win32 @@ -9,6 +13,10 @@ Debug x64 + + Release + ARM64 + Release Win32 @@ -29,23 +37,34 @@ DynamicLibrary Unicode true - v142 + v143 DynamicLibrary Unicode true - v142 + v143 + + + DynamicLibrary + Unicode + true + v143 DynamicLibrary Unicode - v142 + v143 DynamicLibrary Unicode - v142 + v143 + + + DynamicLibrary + Unicode + v143 @@ -56,39 +75,75 @@ + + + + + + <_ProjectFileVersion>10.0.30319.1 $(SolutionDir)$(Configuration)\ $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ $(Configuration)\ $(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ true true + true $(SolutionDir)$(Configuration)\ $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ $(Configuration)\ $(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ false false + false AllRules.ruleset AllRules.ruleset + AllRules.ruleset + + AllRules.ruleset AllRules.ruleset + AllRules.ruleset + + + + + true + + + x64-windows-static-md + + + arm64-windows-static-md + + + x64-windows-static-md + + + x64-windows-static-md + + + x86-windows-static-md @@ -124,6 +179,22 @@ Windows + + + Disabled + WIN32;_DEBUG;_WINDOWS;_USRDLL;HTTPSERVER_EXPORTS;%(PreprocessorDefinitions) + EnableFastChecks + MultiThreadedDebugDLL + + + Level3 + ProgramDatabase + + + true + Windows + + MaxSpeed @@ -163,6 +234,25 @@ true + + + MaxSpeed + true + WIN32;NDEBUG;_WINDOWS;_USRDLL;HTTPSERVER_EXPORTS;%(PreprocessorDefinitions) + MultiThreadedDLL + true + + + Level3 + ProgramDatabase + + + true + Windows + true + true + + diff --git a/install_client_linux.sh b/install_client_linux.sh index 083d5f72c..dcccfb525 100755 --- a/install_client_linux.sh +++ b/install_client_linux.sh @@ -151,6 +151,10 @@ test -e "$PREFIX/sbin" || install -c -m 755 -d "$PREFIX/sbin" test -e "$PREFIX/bin" || install -c -m 755 -d "$PREFIX/bin" install -c "$TARGET/urbackupclientbackend" "$PREFIX/sbin" install -c "$TARGET/urbackupclientctl" "$PREFIX/bin" +if test -e "$TARGET/urbackupclient_dmsnaptool" +then + install -c "$TARGET/urbackupclient_dmsnaptool" "$PREFIX/sbin" +fi ORIG_TARGET=$TARGET @@ -188,6 +192,10 @@ if [ $TARGET != $ORIG_TARGET ] then install -c "$TARGET/urbackupclientbackend" "$PREFIX/sbin" install -c "$TARGET/urbackupclientctl" "$PREFIX/bin" + if test -e "$TARGET/urbackupclient_dmsnaptool" + then + install -c "$TARGET/urbackupclient_dmsnaptool" "$PREFIX/sbin" + fi fi if ! "$PREFIX/bin/urbackupclientctl" --version 2>&1 | grep "UrBackup Client Controller" > /dev/null 2>&1 @@ -233,6 +241,10 @@ install -c "dattobd_create_snapshot" "$PREFIX/share/urbackup" install -c "dattobd_remove_snapshot" "$PREFIX/share/urbackup" install -c "dm_create_snapshot" "$PREFIX/share/urbackup" install -c "dm_remove_snapshot" "$PREFIX/share/urbackup" +install -c "dm_create_snapshot_common" "$PREFIX/share/urbackup" +install -c "dm_cremove_snapshot_common" "$PREFIX/share/urbackup" +install -c "dm_create_volume_snapshot" "$PREFIX/share/urbackup" +install -c "dm_remove_volume_snapshot" "$PREFIX/share/urbackup" install -c "filesystem_snapshot_common" "$PREFIX/share/urbackup" test -e "$PREFIX/etc/urbackup/mariadbdump.conf" || install -c "backup_scripts/mariadbdump.conf" "$PREFIX/etc/urbackup" @@ -307,16 +319,26 @@ then SYSTEMD_DIR="" if command -v pkg-config >/dev/null 2>&1 then - SYSTEMD_DIR=`pkg-config systemd --variable=systemdsystemunitdir` + SYSTEMD_DIR=`pkg-config systemd --variable=systemdsystemunitdir || true` fi if [ "x$SYSTEMD_DIR" = x ] then - echo "Cannot find systemd unit dir. Assuming /lib/systemd/system" - SYSTEMD_DIR="/lib/systemd/system" + if test -e "/lib/systemd/system/urbackupclientbackend.service" + then + echo "Updating existing systemd unit at /lib/systemd/system/urbackupclientbackend.service" + SYSTEMD_DIR="/lib/systemd/system" + elif test -e "/usr/lib/systemd/system" + then + echo "Cannot find systemd unit dir. Assuming /usr/lib/systemd/system" + SYSTEMD_DIR="/usr/lib/systemd/system" + else + echo "Cannot find systemd unit dir. Assuming /lib/systemd/system" + SYSTEMD_DIR="/lib/systemd/system" + fi fi - install -c urbackupclientbackend.service $SYSTEMD_DIR + install -m 644 -c urbackupclientbackend.service $SYSTEMD_DIR systemctl enable urbackupclientbackend.service SYSTEMD_DBUS=yes @@ -410,9 +432,17 @@ fi if [ $SILENT = no ] then - if [ -e $PREFIX/etc/urbackup/snapshot.cfg ] || [ -e $PREFIX/etc/urbackup/no_filesystem_snapshot ] + if [ -e "$PREFIX/etc/urbackup/snapshot.cfg" ] || [ -e "$PREFIX/etc/urbackup/no_filesystem_snapshot" ] then - exit 0 + echo "Snapshots already configured. Keep configuration? [Y/n]" + read yn + if [ "x$yn" != "xn" ] + then + exit 0 + else + ! [ -e "$PREFIX/etc/urbackup/snapshot.cfg" ] || rm -f "$PREFIX/etc/urbackup/snapshot.cfg" + ! [ -e "$PREFIX/etc/urbackup/no_filesystem_snapshot" ] || rm -f "$PREFIX/etc/urbackup/no_filesystem_snapshot" + fi fi CENTOS=no @@ -450,35 +480,20 @@ then then if grep 'VERSION="' /etc/os-release | grep "LTS" > /dev/null 2>&1 then - echo "+Detected Ubuntu LTS. Dattobd supported" + echo "+Detected Ubuntu LTS" UBUNTU=yes - DATTO=yes fi elif grep 'NAME="Debian' /etc/os-release > /dev/null 2>&1 then if grep 'PRETTY_NAME="' /etc/os-release | grep "/sid" > /dev/null 2>&1 then - echo "+Detected Debian unstable/sid. Dattobd not supported" + echo "+Detected Debian unstable/sid" else - echo "+Detected Debian stable. Dattobd supported" - DATTO=yes + echo "+Detected Debian stable" fi fi fi - - if [ $CENTOS != no ] - then - echo "+Detected EL/RH $CENTOS. Dattobd supported" - DATTO=yes - fi - - if [ $FEDORA != no ] - then - echo "+Detected Fedora. Dattobd supported" - DATTO=yes - fi - if [ $DATTO = no ] then echo "-dattobd not supported on this system" @@ -513,7 +528,24 @@ then else echo "-dmsetup not present" fi - + + if [ $DMSETUP != no ] + then + GRUBF=/boot/grub/grub.cfg + if [ -e $GRUBF ] + then + if grep "root=UUID=" $GRUBF > /dev/null 2>&1 || grep "root=PARTUUID=" $GRUBF > /dev/null 2>&1 + then + echo "+Grub is searching for boot device via UUID" + else + echo "-Grub not searching for boot device via UUID. Disabling dmsetup snapshot option" + DMSETUP=no + fi + else + echo "-grub.cfg not found in /boot/grub. Disabling dmsetup snapshot option" + DMSETUP=no + fi + fi while true do @@ -538,7 +570,9 @@ then echo "4) Linux device mapper based snapshots (supports image backups and changed block tracking)" fi - echo "5) Use no snapshot mechanism" + echo "5) Use no snapshot mechanism. Files will be backed up without creating snapshots of them. Images backups will be not supported." + + echo -n "Enter choice (number 1-5, then enter): " read snapn @@ -617,13 +651,15 @@ then then CREATE_SNAPSHOT_SCRIPT="$PREFIX/share/urbackup/dm_create_snapshot" REMOVE_SNAPSHOT_SCRIPT="$PREFIX/share/urbackup/dm_remove_snapshot" - CREATE_VOLUME_SNAPSHOT="$PREFIX/share/urbackup/dm_create_snapshot" - REMOVE_VOLUME_SNAPSHOT="$PREFIX/share/urbackup/dm_remove_snapshot" + CREATE_VOLUME_SNAPSHOT="$PREFIX/share/urbackup/dm_create_volume_snapshot" + REMOVE_VOLUME_SNAPSHOT="$PREFIX/share/urbackup/dm_remove_volume_snapshot" if [ $DEBIAN = yes ] || [ $UBUNTU = yes ] then echo "## Install thin-provisioning tools for changed block detection and partclone for used space optimization ##" apt-get install thin-provisioning-tools partclone || true + echo "## Install libdevmapper1.02.1 for snapshotting device" + apt-get install libdevmapper1.02.1 fi echo "Convert root device into device mapper device on boot (initramfs)? This is required for root device/filesystem backup. [Y/n]" diff --git a/linux_snapshot/apfs_create_snapshot b/linux_snapshot/apfs_create_snapshot new file mode 100755 index 000000000..895a65512 --- /dev/null +++ b/linux_snapshot/apfs_create_snapshot @@ -0,0 +1,40 @@ +#!/bin/sh + +set -e + +SNAP_ID=$1 +SNAP_MOUNTPOINT="$2" +SNAP_DEST=/Volumes/urbackup_snaps/$SNAP_ID + +if [ "x$SNAP_MOUNTPOINT" != "x/" ] +then + echo "APFS snapshots can only be created for root mountpoint /" + exit 1 +fi + +# / seems to be the Data volume +SNAP_SEARCH_MOUNTPOINT="/System/Volumes/Data" + +DEVICE=$(df -P | grep -E " ${SNAP_SEARCH_MOUNTPOINT}\$" | head -n 1 | tr -s " " | cut -d" " -f1) + +echo "Snapshotting device $DEVICE via apfs..." + +SNAP_DATE=$(tmutil localsnapshot | grep "date:" | sed 's/.*: //') +if [ "y$SNAP_DATE" = "y" ]; then + echo "Failed to create snapshot" + exit 1 +fi + +FULL_SNAP_NAME=$(tmutil listlocalsnapshots "$SNAP_MOUNTPOINT" | grep "$SNAP_DATE" | tail -n 1) +if [ "y$FULL_SNAP_NAME" = "y" ]; then + echo "Failed to get full snapshot name" + exit 1 +fi + +mkdir -p "$SNAP_DEST" +echo "$SNAP_DATE" > "$SNAP_DEST-snap-date" +mount_apfs -o rdonly,noatime,nobrowse -s "$FULL_SNAP_NAME" "$DEVICE" "$SNAP_DEST" + +echo "Snapshot mounted at $SNAP_DEST" + +echo "SNAPSHOT=$SNAP_DEST" \ No newline at end of file diff --git a/linux_snapshot/apfs_remove_snapshot b/linux_snapshot/apfs_remove_snapshot new file mode 100755 index 000000000..32ea930ed --- /dev/null +++ b/linux_snapshot/apfs_remove_snapshot @@ -0,0 +1,38 @@ +#!/bin/sh + +set -e + +SNAP_MOUNTPOINT="$2" + +remove_snap() { + ! [ -e "$SNAP_MOUNTPOINT" ] || rmdir "$SNAP_MOUNTPOINT" + + if [ -e "$SNAP_MOUNTPOINT-snap-date" ] + then + SNAP_DATE=$(cat "$SNAP_MOUNTPOINT-snap-date") + tmutil deletelocalsnapshots "$SNAP_DATE" || true + rm "$SNAP_MOUNTPOINT-snap-date" + fi +} + +if [ -e "$SNAP_MOUNTPOINT" ] +then + if ! err=$(umount "$SNAP_MOUNTPOINT" 2>&1) + then + echo "Failed to unmount snapshot at $SNAP_MOUNTPOINT: $err" + + if echo "$err" | grep -q "not currently mounted" + then + echo "Snapshot already unmounted. Removing snapshot directory..." + remove_snap + exit 0 + fi + + lsof | grep "$SNAP_MOUNTPOINT" || true + + sleep 10 + umount "$SNAP_MOUNTPOINT" + fi +fi + +remove_snap \ No newline at end of file diff --git a/linux_snapshot/dattobd_create_snapshot b/linux_snapshot/dattobd_create_snapshot index 986ca5b70..70c3fe55a 100755 --- a/linux_snapshot/dattobd_create_snapshot +++ b/linux_snapshot/dattobd_create_snapshot @@ -15,12 +15,17 @@ SNAP_COWFILE_PATH="/mnt/urbackup_snaps/cbt_info/$SNAP_MOUNTPOINT_SAN-cowfile" exists() { - [ -e $1 ] + [ -e "$1" ] } has_num () { - exists "/mnt/urbackup_snaps/cbt_info/*-snapdev" && grep "$1" "/mnt/urbackup_snaps/cbt_info/*-snapdev" > /dev/null + exists /mnt/urbackup_snaps/cbt_info/*-snapdev && grep "$1" /mnt/urbackup_snaps/cbt_info/*-snapdev > /dev/null +} + +get_path() +{ + echo $1/$2 | sed 's@//@/@g' } @@ -51,11 +56,13 @@ then exit 1 fi +add_to_updatedb_conf "/mnt/urbackup_snaps" + echo "Snapshotting device $DEVICE via dattobd..." NUM=0 -while [ -e "/dev/datto$NUM" ] && has_num $NUM +while [ -e "/dev/datto$NUM" ] || has_num $NUM do NUM=`expr $NUM + 1` done @@ -74,10 +81,10 @@ then TRY_TRANS=1 TRANS_NUM=$(cat "$SNAP_NUM_PATH") echo "Trying to transition /dev/datto$TRANS_NUM to snapshot..." - echo "dbdctl transition-to-snapshot '$SNAP_MOUNTPOINT/.datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID' $TRANS_NUM" + echo "dbdctl transition-to-snapshot $(get_path $SNAP_MOUNTPOINT "datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID") $TRANS_NUM" fi -if [ $TRY_TRANS = 1 ] && dbdctl transition-to-snapshot "$SNAP_MOUNTPOINT/.datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID" $TRANS_NUM +if [ $TRY_TRANS = 1 ] && dbdctl transition-to-snapshot $(get_path $SNAP_MOUNTPOINT ".datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID") $TRANS_NUM then echo "Transitioned /dev/datto$TRANS_NUM to snapshot." CBT_FILE=$(cat $SNAP_COWFILE_PATH) @@ -94,26 +101,26 @@ else echo "Using /dev/datto$NUM..." echo "CBT=type=datto&reset=1" - dbdctl setup-snapshot "$DEVICE" "$SNAP_MOUNTPOINT/.datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID" $NUM + dbdctl setup-snapshot "$DEVICE" $(get_path $SNAP_MOUNTPOINT ".datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID") $NUM fi echo $NUM > ${SNAP_DEST}-num DEV_SIZE=`blockdev --getsize /dev/datto$NUM` -truncate -s100M $SNAP_MOUNTPOINT/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID +truncate -s100M $(get_path $SNAP_MOUNTPOINT ".overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID") LODEV=`losetup -f` if [ "x$LODEV" = x ] then rm "${SNAP_DEST}-num" - rm $SNAP_MOUNTPOINT/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID + rm $(get_path $SNAP_MOUNTPOINT ".overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID") dbdctl destroy $NUM exit 1 fi -losetup $LODEV $SNAP_MOUNTPOINT/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID +losetup $LODEV $(get_path $SNAP_MOUNTPOINT ".overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID") echo "0 $DEV_SIZE snapshot /dev/datto$NUM $LODEV N 8" | dmsetup create "wsnap-$SNAP_ID" @@ -136,14 +143,14 @@ then rm "${SNAP_DEST}-num" dmsetup remove "wsnap-$SNAP_ID" losetup -d $LODEV - rm $SNAP_MOUNTPOINT/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID + rm $(get_path $SNAP_MOUNTPOINT ".overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID") dbdctl destroy $NUM exit 1 fi echo "/dev/datto$NUM" > ${SNAP_DEST}-dev echo "$NUM" > $SNAP_NUM_PATH -echo "$SNAP_MOUNTPOINT/.datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID" > $SNAP_COWFILE_PATH +echo $(get_path $SNAP_MOUNTPOINT ".datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID") > $SNAP_COWFILE_PATH echo "SNAPSHOT=$SNAP_DEST" exit 0 diff --git a/linux_snapshot/dattobd_remove_snapshot b/linux_snapshot/dattobd_remove_snapshot index ac85a4223..dd6f900c0 100755 --- a/linux_snapshot/dattobd_remove_snapshot +++ b/linux_snapshot/dattobd_remove_snapshot @@ -9,22 +9,27 @@ SNAP_ORIG_PATH_SAN=$(echo "$SNAP_ORIG_PATH" | sed 's@/@_@g') CDIR=`dirname $0` +get_path() +{ + echo $1/$2 | sed 's@//@/@g' +} + remove_overlay() { - if test -e "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID" + if test -e $(get_path "$SNAP_ORIG_PATH" "/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID") then - LODEV=`losetup -j "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID" | cut -d':' -f1` + LODEV=`losetup -j $(get_path "$SNAP_ORIG_PATH" "/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID") | cut -d':' -f1` if [ "x$LODEV" != x ] then losetup -d $LODEV fi - rm "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID" + rm $(get_path "$SNAP_ORIG_PATH" "/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID") fi } if ! test -e $SNAP_MOUNTPOINT then echo "Snapshot at $SNAP_MOUNTPOINT was already removed" - [ ! -e "$SNAP_ORIG_PATH/.datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID" ] || rm "$SNAP_ORIG_PATH/.datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID" + [ ! -e $(get_path "$SNAP_ORIG_PATH" "/.datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID") ] || rm $(get_path "$SNAP_ORIG_PATH" "/.datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID") [ ! -e "${SNAP_MOUNTPOINT}-num" ] || rm "${SNAP_MOUNTPOINT}-num" remove_overlay exit 0 @@ -51,7 +56,7 @@ then echo "Snapshot is not mounted. Already removed" rm "${SNAP_MOUNTPOINT}-num" rmdir "${SNAP_MOUNTPOINT}" - [ ! -e "$SNAP_ORIG_PATH/.datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID" ] || rm "$SNAP_ORIG_PATH/.datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID" + [ ! -e $(get_path "$SNAP_ORIG_PATH" "/.datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID") ] || rm $(get_path "$SNAP_ORIG_PATH" "/.datto_3d41c58e-6724-4d47-8981-11c766a08a24_$SNAP_ID") remove_overlay exit 0 fi diff --git a/linux_snapshot/dm_create_snapshot b/linux_snapshot/dm_create_snapshot index f0575fc0c..eff886b1b 100755 --- a/linux_snapshot/dm_create_snapshot +++ b/linux_snapshot/dm_create_snapshot @@ -2,175 +2,5 @@ set -e -mkdir -p /mnt/urbackup_snaps - CDIR=`dirname $0` - -SNAP_ID=$1 -SNAP_MOUNTPOINT="$2" -SNAP_DEST=/mnt/urbackup_snaps/$SNAP_ID - -CDIR=`dirname $0` -. $CDIR/filesystem_snapshot_common -exit_exclude_snapshot_mountpoints "$SNAP_MOUNTPOINT" - -DEVICE=$(df -T -P | egrep " ${SNAP_MOUNTPOINT}\$" | head -n 1 | tr -s " " | cut -d" " -f1) -set_filesystem_type "${SNAP_MOUNTPOINT}" - -if [ "x$TYPE" = "x" ] -then - if btrfs subvolume list -o "$SNAP_MOUNTPOINT" > /dev/null 2>&1 - then - TYPE="btrfs" - fi -fi - -if [ "x$TYPE" = "xbtrfs" ] -then - $CDIR/btrfs_create_filesystem_snapshot "$@" - exit $? -fi - -if [ "x$DEVICE" = "x" ] -then - echo "Cannot get device for filesystem $SNAP_MOUNTPOINT" - exit 1 -fi - -echo "Snapshotting device $DEVICE via dm..." - -if ! dmsetup table "$DEVICE" > /dev/null 2>&1 -then - echo "$DEVICE is not a device mapper device. Cannot snapshot via dm." - exit 1 -fi - -modprobe dm_snapshot - -DEV_SIZE=$(blockdev --getsz "$DEVICE") - -if [ "x$DEV_SIZE" = "x" ] -then - echo "Cannot get device size of device $DEVICE" - exit 1 -fi - -DEVNAME=$(basename $DEVICE) - -RUUID="a31725acca86421d" - -ORIG_DEVICE="/dev/mapper/$DEVNAME-$RUUID-clone" -ERA_META_FN=".era-meta_3d41c58e-6724-4d47-8981-11c766a08a24" - -ERA_RESET=0 -if ! command -v era_dump >/dev/null 2>&1 -then - echo "thin-provisioning-tools not installed. CBT not enabled." -elif modprobe dm_era && ! dmsetup table "$DEVICE" | grep " era " > /dev/null 2>&1 && ! dmsetup table "$DEVICE" | grep " snapshot-origin " > /dev/null 2>&1 -then - echo "Layering in dm-era device..." - META_SIZE=$(( (((DEV_SIZE*4)/1024)/512)*512 + 1*1024*1024 )) - if [ $META_SIZE -lt $((2*1024*1024)) ] - then - META_SIZE=$((2*1024*1024)) - fi - - if [ -e "$SNAP_MOUNTPOINT/$ERA_META_FN" ] - then - chattr -i "$SNAP_MOUNTPOINT/$ERA_META_FN" || true - rm "$SNAP_MOUNTPOINT/$ERA_META_FN" - fi - - fallocate -l $META_SIZE "$SNAP_MOUNTPOINT/$ERA_META_FN" - - ORIG_DEVICE="/dev/mapper/$DEVNAME-$RUUID-clone-era" - dmsetup table "$DEVICE" | dmsetup create "$DEVNAME-$RUUID-clone-era" - urbackupclientbackend --internal --print-dm-file-extents "$SNAP_MOUNTPOINT/$ERA_META_FN" --file-dm-block-dev "$ORIG_DEVICE" | dmsetup create "$DEVNAME-$RUUID-era-metadata" - dmsetup table "$DEVNAME-$RUUID-era-metadata" | dmsetup create "$DEVNAME-$RUUID-era-metadata-access" - dd if=/dev/zero of=/dev/mapper/$DEVNAME-$RUUID-era-metadata bs=512 count=$((META_SIZE/512)) > /dev/null 2>&1 - echo "0 $DEV_SIZE era /dev/mapper/$DEVNAME-$RUUID-era-metadata $ORIG_DEVICE 1024" | dmsetup create "$DEVNAME-$RUUID-clone" - echo "CBT=type=era&reset=1" - ERA_RESET=1 -fi - -OVERLAY_FN="$SNAP_MOUNTPOINT/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID" -if [ -e "$OVERLAY_FN" ] -then - chattr -i "$OVERLAY_FN" || true - rm "$OVERLAY_FN" -fi - -fallocate -l 5G "$OVERLAY_FN" - -if ! [ -e "/dev/mapper/$DEVNAME-$RUUID-clone" ] -then - dmsetup table "$DEVICE" | dmsetup create "$DEVNAME-$RUUID-clone" -fi - -urbackupclientbackend --internal --print-dm-file-extents "$OVERLAY_FN" --file-dm-block-dev "$ORIG_DEVICE" | dmsetup create "$DEVNAME-$SNAP_ID-cow-storage" -UDEVCOOKIE=$(dmsetup udevcreatecookie) -dmsetup suspend "$DEVICE" - -if dmsetup table "$DEVNAME-$RUUID-clone" | grep " era " > /dev/null 2>&1 -then - echo "Checkpointing device for CBT..." - dmsetup message "$DEVNAME-$RUUID-clone" 0 checkpoint - if [ $ERA_RESET != 1 ] - then - echo "CBT=type=era" - echo "CBT_FILE=/dev/mapper/$DEVNAME-$RUUID-era-metadata-access" - fi -fi - -echo "Creating snapshot device..." -echo "0 $DEV_SIZE snapshot /dev/mapper/$DEVNAME-$RUUID-clone /dev/mapper/$DEVNAME-$SNAP_ID-cow-storage N 32" | dmsetup create -r --udevcookie "$UDEVCOOKIE" "$DEVNAME-$SNAP_ID" - -if ! [ -e "/dev/mapper/$DEVNAME-$RUUID-origin" ] -then - echo "Creating snapshot origin device..." - echo "0 $DEV_SIZE snapshot-origin /dev/mapper/$DEVNAME-$RUUID-clone" | dmsetup create --udevcookie "$UDEVCOOKIE" "$DEVNAME-$RUUID-origin" -fi - -if ! dmsetup table "$DEVICE" | grep " snapshot-origin " > /dev/null 2>&1 -then - dmsetup table "$DEVNAME-$RUUID-origin" | dmsetup load "$DEVICE" -fi - -dmsetup resume "$DEVICE" -dmsetup udevreleasecookie "$UDEVCOOKIE" - -truncate -s100M $SNAP_MOUNTPOINT/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID-wsnap - -LODEV=`losetup -f` - -if [ "x$LODEV" = x ] -then - echo "TODO: Cleanup" - exit 1 -fi - -losetup $LODEV $SNAP_MOUNTPOINT/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID-wsnap - -echo "0 $DEV_SIZE snapshot /dev/mapper/$DEVNAME-$SNAP_ID $LODEV N 8" | dmsetup create "$DEVNAME-$SNAP_ID-wsnap" - -echo "Mounting /dev/mapper/$DEVNAME-$SNAP_ID-wsnap..." - -MOUNTOPTS="ro" - -if [ $TYPE = "xfs" ] -then - MOUNTOPTS="ro,nouuid" -fi - -mkdir -p "$SNAP_DEST" - -if ! mount -o $MOUNTOPTS "/dev/mapper/$DEVNAME-$SNAP_ID-wsnap" "$SNAP_DEST" -then - echo "Mounting filesystem failed" - #TODO: CLEANUP - exit 1 -fi - -echo "$DEVNAME" > ${SNAP_DEST}-name -echo "/dev/mapper/$DEVNAME-$SNAP_ID" > ${SNAP_DEST}-dev -echo "SNAPSHOT=$SNAP_DEST" +$CDIR/dm_create_snapshot_common fs "$@" \ No newline at end of file diff --git a/linux_snapshot/dm_create_snapshot_common b/linux_snapshot/dm_create_snapshot_common new file mode 100755 index 000000000..7ed4db7cc --- /dev/null +++ b/linux_snapshot/dm_create_snapshot_common @@ -0,0 +1,197 @@ +#!/bin/sh + +set -e + +mkdir -p /mnt/urbackup_snaps + +CDIR=`dirname $0` + +VOLUME_SNAP=0 +if [ "x$1" = "xvol" ] +then + VOLUME_SNAP=1 +elif [ "x$1" != "xfs" ] +then + echo "First parameter must be either 'fs' or 'vol'" + exit 1 +fi +shift + +SNAP_ID=$1 +SNAP_MOUNTPOINT="$2" +SNAP_DEST=/mnt/urbackup_snaps/$SNAP_ID + +CDIR=`dirname $0` +. $CDIR/filesystem_snapshot_common +exit_exclude_snapshot_mountpoints "$SNAP_MOUNTPOINT" + +DEVICE=$(df -T -P | egrep " ${SNAP_MOUNTPOINT}\$" | head -n 1 | tr -s " " | cut -d" " -f1) +set_filesystem_type "${SNAP_MOUNTPOINT}" +DEVNAME=$(basename $DEVICE) + + +if [ "x$TYPE" = "x" ] +then + if btrfs subvolume list -o "$SNAP_MOUNTPOINT" > /dev/null 2>&1 + then + TYPE="btrfs" + fi +fi + +if [ $VOLUME_SNAP = 0 ] && [ "x$TYPE" = "xbtrfs" ] +then + $CDIR/btrfs_create_filesystem_snapshot "$@" + exit $? +fi + +if [ "x$TYPE" = "xvfat" ] +then + mount -o remount,ro "$SNAP_MOUNTPOINT" + echo "$DEVICE" > "$SNAP_MOUNTPOINT-dev" + echo "vfat" > "$SNAP_MOUNTPOINT-name" + echo "SNAPSHOT=$SNAP_MOUNTPOINT" + exit 0 +fi + + +if [ "x$TYPE" != "xxfs" ] && [ "x$TYPE" != "xext4" ] && [ "x$TYPE" != "xbtrfs" ] +then + echo "File system $TYPE not supported" + exit 1 +fi + +if [ "x$DEVICE" = "x" ] +then + echo "Cannot get device for filesystem $SNAP_MOUNTPOINT" + exit 1 +fi + +echo "Snapshotting device $DEVICE via dm..." + +if ! dmsetup table "$DEVICE" > /dev/null 2>&1 +then + echo "$DEVICE is not a device mapper device. Cannot snapshot via dm." + exit 1 +fi + +modprobe dm_snapshot + +DEV_SIZE=$(blockdev --getsz "$DEVICE") + +if [ "x$DEV_SIZE" = "x" ] +then + echo "Cannot get device size of device $DEVICE" + exit 1 +fi + + +RUUID="a31725acca86421d" + +ORIG_DEVICE="/dev/mapper/$DEVNAME-$RUUID-clone" +ERA_META_FN=".era-meta_3d41c58e-6724-4d47-8981-11c766a08a24" + +ERA_RESET=0 +if ! command -v era_dump >/dev/null 2>&1 +then + echo "thin-provisioning-tools not installed. CBT not enabled." +elif modprobe dm_era && ! dmsetup table "$DEVICE" | grep " era " > /dev/null 2>&1 && ! dmsetup table "$DEVICE" | grep " snapshot-origin " > /dev/null 2>&1 +then + echo "Layering in dm-era device..." + META_SIZE=$(( (((DEV_SIZE*4)/1024)/512)*512 + 3*1024*1024 )) + if [ $META_SIZE -lt $((4*1024*1024)) ] + then + META_SIZE=$((4*1024*1024)) + fi + + if [ -e "$SNAP_MOUNTPOINT/$ERA_META_FN" ] + then + chattr -i "$SNAP_MOUNTPOINT/$ERA_META_FN" || true + rm "$SNAP_MOUNTPOINT/$ERA_META_FN" + fi + + if [ "$TYPE" = "btrfs" ] + then + touch "$SNAP_MOUNTPOINT/$ERA_META_FN" + chattr +C "$SNAP_MOUNTPOINT/$ERA_META_FN" + fi + + fallocate -l $META_SIZE "$SNAP_MOUNTPOINT/$ERA_META_FN" + chattr +i "$SNAP_MOUNTPOINT/$ERA_META_FN" + echo "FLOCK_PERM=$SNAP_MOUNTPOINT/$ERA_META_FN" + + ORIG_DEVICE="/dev/mapper/$DEVNAME-$RUUID-clone-era" + dmsetup table "$DEVICE" | dmsetup create "$DEVNAME-$RUUID-clone-era" + urbackupclientbackend --internal --print-dm-file-extents "$SNAP_MOUNTPOINT/$ERA_META_FN" --file-dm-block-dev "$ORIG_DEVICE" | dmsetup create "$DEVNAME-$RUUID-era-metadata" + dmsetup table "$DEVNAME-$RUUID-era-metadata" | dmsetup create "$DEVNAME-$RUUID-era-metadata-access" + dd if=/dev/zero of=/dev/mapper/$DEVNAME-$RUUID-era-metadata bs=512 count=$(blockdev --getsz /dev/mapper/$DEVNAME-$RUUID-era-metadata) > /dev/null 2>&1 + echo "0 $DEV_SIZE era /dev/mapper/$DEVNAME-$RUUID-era-metadata $ORIG_DEVICE 1024" | dmsetup create "$DEVNAME-$RUUID-clone" + echo "CBT=type=era&reset=1" + ERA_RESET=1 +fi + +OVERLAY_FN="$SNAP_MOUNTPOINT/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID" +if [ -e "$OVERLAY_FN" ] +then + chattr -i "$OVERLAY_FN" || true + rm "$OVERLAY_FN" +fi + +if [ "$TYPE" = "btrfs" ] +then + touch "$OVERLAY_FN" + chattr +C "$OVERLAY_FN" +fi + +fallocate -l 5G "$OVERLAY_FN" +chattr +i "$OVERLAY_FN" +echo "FLOCK=$OVERLAY_FN" + +if ! [ -e "/dev/mapper/$DEVNAME-$RUUID-clone" ] +then + dmsetup table "$DEVICE" | dmsetup create "$DEVNAME-$RUUID-clone" +fi + +urbackupclientbackend --internal --print-dm-file-extents "$OVERLAY_FN" --file-dm-block-dev "$ORIG_DEVICE" | dmsetup create "$DEVNAME-$SNAP_ID-cow-storage" +# Needs to run with mlockall for root device -> cannot do that in bash +urbackupclient_dmsnaptool --dev "$DEVICE" --clone-dev "/dev/mapper/$DEVNAME-$RUUID-clone" --snap-dev "/dev/mapper/$DEVNAME-$SNAP_ID" --cow-dev "/dev/mapper/$DEVNAME-$SNAP_ID-cow-storage" --origin-dev "/dev/mapper/$DEVNAME-$RUUID-origin" --era-access-dev "/dev/mapper/$DEVNAME-$RUUID-era-metadata-access" --dev-size "$DEV_SIZE" + +truncate -s100M $SNAP_MOUNTPOINT/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID-wsnap + +LODEV=`losetup -f` + +if [ "x$LODEV" = x ] +then + echo "TODO: Cleanup" + exit 1 +fi + +losetup $LODEV $SNAP_MOUNTPOINT/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID-wsnap + +echo "0 $DEV_SIZE snapshot /dev/mapper/$DEVNAME-$SNAP_ID $LODEV N 8" | dmsetup create "$DEVNAME-$SNAP_ID-wsnap" + +echo "Mounting /dev/mapper/$DEVNAME-$SNAP_ID-wsnap..." + +MOUNTOPTS="ro" + +if [ $TYPE = "xfs" ] +then + MOUNTOPTS="ro,nouuid" +elif [ $TYPE = "btrfs" ] && [ $VOLUME_SNAP = 0 ] +then + btrfstune -m "/dev/mapper/$DEVNAME-$SNAP_ID-wsnap" +fi + +mkdir -p "$SNAP_DEST" +if [ $VOLUME_SNAP = 0 ] +then + if ! mount -o $MOUNTOPTS "/dev/mapper/$DEVNAME-$SNAP_ID-wsnap" "$SNAP_DEST" + then + echo "Mounting filesystem failed" + #TODO: CLEANUP + exit 1 + fi +fi + +echo "$DEVNAME" > ${SNAP_DEST}-name +echo "/dev/mapper/$DEVNAME-$SNAP_ID" > ${SNAP_DEST}-dev +echo "SNAPSHOT=$SNAP_DEST" diff --git a/linux_snapshot/dm_create_volume_snapshot b/linux_snapshot/dm_create_volume_snapshot new file mode 100755 index 000000000..89dcdcae4 --- /dev/null +++ b/linux_snapshot/dm_create_volume_snapshot @@ -0,0 +1,6 @@ +#!/bin/sh + +set -e + +CDIR=`dirname $0` +$CDIR/dm_create_snapshot_common vol "$@" \ No newline at end of file diff --git a/linux_snapshot/dm_remove_snapshot b/linux_snapshot/dm_remove_snapshot index 4863784db..4189ab016 100755 --- a/linux_snapshot/dm_remove_snapshot +++ b/linux_snapshot/dm_remove_snapshot @@ -2,91 +2,5 @@ set -e -SNAP_ID=$1 -SNAP_MOUNTPOINT="$2" -SNAP_ORIG_PATH="$5" - CDIR=`dirname $0` - -remove_overlay() { - if test -e "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID" - then - chattr -i "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID" - rm "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID" - fi - - if test -e "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID-wsnap" - then - LODEV=`losetup -j "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID-wsnap" | cut -d':' -f1` - if [ "x$LODEV" != x ] - then - losetup -d "$LODEV" - fi - rm "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID-wsnap" - fi -} - -if ! test -e $SNAP_MOUNTPOINT -then - echo "Snapshot at $SNAP_MOUNTPOINT was already removed" - remove_overlay - exit 0 -fi - -TYPE=$(df -T -P | egrep " ${SNAP_MOUNTPOINT}\$" | head -n 1 | tr -s " " | cut -d" " -f2) - -if [ "x$TYPE" = "x" ] -then - if btrfs subvolume list -o "$SNAP_MOUNTPOINT" > /dev/null 2>&1 - then - TYPE="btrfs" - fi -fi - -if [ "x$TYPE" = "xbtrfs" ] -then - $CDIR/btrfs_remove_filesystem_snapshot "$@" - exit $? -fi - -if ! df -T -P | egrep " ${SNAP_MOUNTPOINT}\$" > /dev/null 2>&1 -then - echo "Snapshot is not mounted. Already removed" - remove_overlay - exit 0 -fi - -if ! [ -e "${SNAP_MOUNTPOINT}-name" ] -then - echo "Could not find snapshot device name at ${SNAP_MOUNTPOINT}-name" - remove_overlay - exit 1 -fi - -DEVNAME=$(cat "${SNAP_MOUNTPOINT}-name") - -echo "Unmounting /dev/mapper/$DEVNAME-$SNAP_ID at /mnt/urbackup_snaps/$SNAP_ID..." - -if ! umount /mnt/urbackup_snaps/$SNAP_ID -then - lsof | grep /mnt/urbackup_snaps/$SNAP_ID || true - echo "Unmounting /mnt/urbackup_snaps/$SNAP_ID failed. Retrying in 10s..." - sleep 10 - umount /mnt/urbackup_snaps/$SNAP_ID -fi - -rm "${SNAP_MOUNTPOINT}-name" -rm "${SNAP_MOUNTPOINT}-dev" -rmdir "${SNAP_MOUNTPOINT}" - -echo "Removing dm snapshot..." - -dmsetup remove "$DEVNAME-$SNAP_ID-wsnap" -dmsetup remove "$DEVNAME-$SNAP_ID" - -echo "Removing snapshot cow storage..." -dmsetup remove "$DEVNAME-$SNAP_ID-cow-storage" - -remove_overlay - -exit 0 +$CDIR/dm_remove_snapshot_common fs "$@" \ No newline at end of file diff --git a/linux_snapshot/dm_remove_snapshot_common b/linux_snapshot/dm_remove_snapshot_common new file mode 100755 index 000000000..8565d4c21 --- /dev/null +++ b/linux_snapshot/dm_remove_snapshot_common @@ -0,0 +1,148 @@ +#!/bin/sh + +set -e + +VOLUME_SNAP=0 +if [ "x$1" = "xvol" ] +then + VOLUME_SNAP=1 +elif [ "x$1" != "xfs" ] +then + echo "First parameter must be either 'fs' or 'vol'" + exit 1 +fi +shift + +SNAP_ID=$1 +SNAP_MOUNTPOINT="$2" +SNAP_ORIG_PATH="$5" + +CDIR=`dirname $0` + +remove_overlay() { + if test -e "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID" + then + chattr -i "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID" + rm "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID" + fi + + if test -e "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID-wsnap" + then + LODEV=`losetup -j "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID-wsnap" | cut -d':' -f1` + if [ "x$LODEV" != x ] + then + losetup -d "$LODEV" + fi + rm "$SNAP_ORIG_PATH/.overlay_2fefd007-3e48-4162-b2c6-45ccdda22f37_$SNAP_ID-wsnap" + fi +} + +remove_dm() { + if ! [ -e "${SNAP_MOUNTPOINT}-name" ] + then + if [ $VOLUME_SNAP = 1 ] + then + TYPE=$(df -T -P | egrep " ${SNAP_MOUNTPOINT}\$" | head -n 1 | tr -s " " | cut -d" " -f2) + if [ "x$TYPE" = "xvfat" ] + then + echo "Snapshot is an vfat partition and no name found. No dm nodes to remove" + return 0 + fi + fi + + echo "Could not find snapshot device name at ${SNAP_MOUNTPOINT}-name. Cannot remove dm nodes" + return 1 + fi + + DEVNAME=$(cat "${SNAP_MOUNTPOINT}-name") + + if [ "x$DEVNAME" = "xvfat" ] + then + echo "Snapshot is an vfat partition. No dm nodes to remove" + rm "${SNAP_MOUNTPOINT}-name" || true + rm "${SNAP_MOUNTPOINT}-dev" || true + return 0 + fi + + echo "Removing dm snapshot..." + + dmsetup remove "$DEVNAME-$SNAP_ID-wsnap" || true + dmsetup remove "$DEVNAME-$SNAP_ID" || true + + echo "Removing snapshot cow storage..." + dmsetup remove "$DEVNAME-$SNAP_ID-cow-storage" || true + + rm "${SNAP_MOUNTPOINT}-name" + rm "${SNAP_MOUNTPOINT}-dev" + rmdir "${SNAP_MOUNTPOINT}" +} + +if ! test -e $SNAP_MOUNTPOINT +then + echo "Snapshot at $SNAP_MOUNTPOINT was already removed" + remove_dm + remove_overlay + exit 0 +fi + +TYPE=$(df -T -P | egrep " ${SNAP_MOUNTPOINT}\$" | head -n 1 | tr -s " " | cut -d" " -f2) + +if [ "x$TYPE" = "x" ] +then + if [ $VOLUME_SNAP = 1 ] + then + echo "Removing volume snapshot $SNAP_MOUNTPOINT..." + remove_dm + remove_overlay + exit 0 + fi + + if btrfs subvolume list -o "$SNAP_MOUNTPOINT" > /dev/null 2>&1 + then + TYPE="btrfs" + fi +fi + +if [ "x$TYPE" = "xbtrfs" ] && [ $VOLUME_SNAP = 0 ] +then + $CDIR/btrfs_remove_filesystem_snapshot "$@" + exit $? +fi + +if ! df -T -P | egrep " ${SNAP_MOUNTPOINT}\$" > /dev/null 2>&1 +then + echo "Snapshot is not mounted. Already removed" + remove_dm + remove_overlay + exit 0 +fi + +if [ "x$TYPE" = "xvfat" ] +then + mount -o remount,rw "$SNAP_MOUNTPOINT" + remove_dm + remove_overlay + exit 0 +fi + +if ! [ -e "${SNAP_MOUNTPOINT}-name" ] +then + echo "Could not find snapshot device name at ${SNAP_MOUNTPOINT}-name" + exit 1 +fi + +DEVNAME=$(cat "${SNAP_MOUNTPOINT}-name") + +echo "Unmounting /dev/mapper/$DEVNAME-$SNAP_ID at /mnt/urbackup_snaps/$SNAP_ID..." + +if ! umount /mnt/urbackup_snaps/$SNAP_ID +then + lsof | grep /mnt/urbackup_snaps/$SNAP_ID || true + echo "Unmounting /mnt/urbackup_snaps/$SNAP_ID failed. Retrying in 10s..." + sleep 10 + umount /mnt/urbackup_snaps/$SNAP_ID +fi + +remove_dm +remove_overlay +exit 0 diff --git a/linux_snapshot/dm_remove_volume_snapshot b/linux_snapshot/dm_remove_volume_snapshot new file mode 100755 index 000000000..579ed1916 --- /dev/null +++ b/linux_snapshot/dm_remove_volume_snapshot @@ -0,0 +1,6 @@ +#!/bin/sh + +set -e + +CDIR=`dirname $0` +$CDIR/dm_remove_snapshot_common vol "$@" \ No newline at end of file diff --git a/linux_snapshot/filesystem_snapshot_common b/linux_snapshot/filesystem_snapshot_common index 838f265aa..6fff3a4d7 100755 --- a/linux_snapshot/filesystem_snapshot_common +++ b/linux_snapshot/filesystem_snapshot_common @@ -21,3 +21,13 @@ set_filesystem_type() { TYPE=`df -T -P | egrep " ${1}\$" | head -n 1 | tr -s " " | cut -d" " -f2` export TYPE } + +add_to_updatedb_conf() { + if test -e /etc/updatedb.conf + then + if ! grep -q "$1" /etc/updatedb.conf + then + sed -i "s|^PRUNEPATHS=\"|PRUNEPATHS=\"$1 |" /etc/updatedb.conf + fi + fi +} \ No newline at end of file diff --git a/linux_snapshot/scripts_local-top_urbackup-setup-snapshot b/linux_snapshot/scripts_local-top_urbackup-setup-snapshot index a30c46591..9ea3d271a 100755 --- a/linux_snapshot/scripts_local-top_urbackup-setup-snapshot +++ b/linux_snapshot/scripts_local-top_urbackup-setup-snapshot @@ -21,10 +21,21 @@ if grep "setup-snapshot=0" /proc/cmdline; then exit 0 fi +if [ "x$ROOT" = "x" ]; then + echo "urbackup-setup-snapshot: root device not defined. Not setting up snapshotting." > /dev/kmsg + exit 0 +fi + +if echo "$ROOT" | grep "root-98d1f8b1f435"; then + panic "urbackup-setup-snapshot: Root device is snapshot (clobbered: $ROOT). Please edit root= boot parameter to be the root device and add 'setup-snapshot=0' to boot parameters to fix boot. Then run 'update-grub' to fix grub config." + exit 0 +fi + RBD=$(resolve_device "$ROOT") if [ "x$RBD" = "x" ]; then echo "urbackup-setup-snapshot: root device not found. Not setting up snapshotting." > /dev/kmsg + exit 0 fi echo "urbackup-setup-snapshot: root block device $RBD" > /dev/kmsg diff --git a/logrotate_urbackupsrv b/logrotate_urbackupsrv index b967a8a8a..89a9770be 100644 --- a/logrotate_urbackupsrv +++ b/logrotate_urbackupsrv @@ -5,6 +5,6 @@ create 640 urbackup urbackup compress postrotate - test -e /var/run/urbackupsrv.pid && kill -HUP `cat /var/run/urbackupsrv.pid` || /bin/systemctl kill -s HUP urbackup-server.service + test -e /var/run/urbackupsrv.pid && kill -HUP `cat /var/run/urbackupsrv.pid` || test -e /var/run/urbackupsrv.ready && /bin/systemctl kill -s HUP --kill-who=main urbackup-server.service endscript } diff --git a/luaplugin/luaplugin.vcxproj b/luaplugin/luaplugin.vcxproj index af4ffac05..85f4abf67 100644 --- a/luaplugin/luaplugin.vcxproj +++ b/luaplugin/luaplugin.vcxproj @@ -1,10 +1,18 @@  + + Debug + ARM64 + Debug Win32 + + Release + ARM64 + Release Win32 @@ -28,26 +36,39 @@ DynamicLibrary true - v142 + v143 Unicode DynamicLibrary false - v142 + v143 true Unicode DynamicLibrary true - v142 + v143 + Unicode + + + DynamicLibrary + true + v143 Unicode DynamicLibrary false - v142 + v143 + true + Unicode + + + DynamicLibrary + false + v143 true Unicode @@ -65,9 +86,15 @@ + + + + + + true @@ -75,12 +102,36 @@ true + + true + false false + + false + + + true + + + x64-windows-static-md + + + arm64-windows-static-md + + + x64-windows-static-md + + + x64-windows-static-md + + + x86-windows-static-md + @@ -107,6 +158,19 @@ true + + + + + Level3 + Disabled + _DEBUG;_WINDOWS;_USRDLL;LUAPLUGIN_EXPORTS;%(PreprocessorDefinitions) + + + Windows + true + + Level3 @@ -141,6 +205,23 @@ true + + + Level3 + + + MaxSpeed + true + true + NDEBUG;_WINDOWS;_USRDLL;LUAPLUGIN_EXPORTS;%(PreprocessorDefinitions) + + + Windows + true + true + true + + diff --git a/main.cpp b/main.cpp index fc0587065..ea6122af4 100644 --- a/main.cpp +++ b/main.cpp @@ -437,9 +437,26 @@ int main_fkt(int argc, char *argv[]) } } #endif - - +#ifndef _WIN32 + if (signal(SIGINT, termination_handler) == SIG_IGN) + signal(SIGINT, SIG_IGN); + /*if(!daemon) + { + if (signal (SIGHUP, termination_handler) == SIG_IGN) + signal (SIGHUP, SIG_IGN); + } + else*/ + { + if (signal(SIGHUP, hub_handler) == SIG_IGN) + signal(SIGHUP, SIG_IGN); + } + if (signal(SIGTERM, termination_handler) == SIG_IGN) + signal(SIGTERM, SIG_IGN); +#else + signal(SIGABRT, abort_handler); + _set_invalid_parameter_handler(invalid_parameter_handler); +#endif if(!logfile.empty()) { @@ -560,31 +577,15 @@ int main_fkt(int argc, char *argv[]) Server->createThread(lbs); } - -#ifndef _WIN32 - if (signal (SIGINT, termination_handler) == SIG_IGN) - signal (SIGINT, SIG_IGN); - /*if(!daemon) - { - if (signal (SIGHUP, termination_handler) == SIG_IGN) - signal (SIGHUP, SIG_IGN); - } - else*/ - { - if (signal (SIGHUP, hub_handler) == SIG_IGN) - signal (SIGHUP, SIG_IGN); - } - if (signal (SIGTERM, termination_handler) == SIG_IGN) - signal (SIGTERM, SIG_IGN); -#else - signal(SIGABRT, abort_handler); - _set_invalid_parameter_handler(invalid_parameter_handler); -#endif ((CSessionMgr*)Server->getSessionMgr())->startTimeoutSessionThread(); Server->startupComplete(); +#ifndef _WIN32 + writestring("", "/var/run/urbackupsrv.ready"); +#endif + if(no_server==false ) { init_mutex_selthread(); diff --git a/md5.h b/md5.h index 63807f9c1..202ba8537 100644 --- a/md5.h +++ b/md5.h @@ -117,12 +117,12 @@ class MD5 { #define CRYPTOPP_ENABLE_NAMESPACE_WEAK 1 #ifdef _WIN32 -#include +#define CRYPTOPP_INCLUDE_PREFIX cryptopp #else #include "config.h" +#endif #define CRYPTOPP_INCLUDE_MD5 #include CRYPTOPP_INCLUDE_MD5 -#endif class MD5 { diff --git a/mount_helper/main.cpp b/mount_helper/main.cpp index b8123e158..e5e9cfcf5 100644 --- a/mount_helper/main.cpp +++ b/mount_helper/main.cpp @@ -1,822 +1,963 @@ -#include -#include -#include -#include "../stringtools.h" -#include "../urbackupcommon/os_functions.h" -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef __linux__ -#include -#endif -#include -#include -extern char **environ; - -#define DEF_Server -#include "../Server.h" -#include "../config.h" - -#ifndef LOOP_SET_DIRECT_IO -#define LOOP_SET_DIRECT_IO 0x4C08 -#endif -#ifndef LOOP_CTL_GET_FREE -#define LOOP_CTL_GET_FREE 0x4C82 -#endif -#ifndef O_CLOEXEC -#define O_CLOEXEC 0 -#endif -#define LOCAL_LO_FLAGS_AUTOCLEAR 4 - -IServer *Server; - -const char* mdconfig_path = "/sbin/mdconfig"; -const char* umount_path = "/sbin/umount"; - - -std::string getBackupfolderPath() -{ - std::string fn=trim(getFile("/etc/urbackup/backupfolder")); - if(fn.find("\n")!=std::string::npos) - fn=getuntil("\n", fn); - if(fn.find("\r")!=std::string::npos) - fn=getuntil("\r", fn); - - return fn; -} - -std::string handleFilename(std::string fn) -{ - fn=conv_filename(fn); - if(fn=="..") - { - return ""; - } - return fn; -} - -int exec_wait(const std::string& path, bool keep_stdout, ...) -{ - va_list vl; - va_start(vl, keep_stdout); - - std::vector args; - args.push_back(const_cast(path.c_str())); - - while(true) - { - const char* p = va_arg(vl, const char*); - if(p==NULL) break; - args.push_back(const_cast(p)); - } - va_end(vl); - - args.push_back(NULL); - - pid_t child_pid = fork(); - - if(child_pid==0) - { - environ = new char*[1]; - *environ=NULL; - - if(!keep_stdout) - { - int nullfd = open("/dev/null", O_WRONLY); - - if(nullfd!=-1) - { - if(dup2(nullfd, 1)==-1) - { - return -1; - } - - if(dup2(nullfd, 2)==-1) - { - return -1; - } - } - else - { - return -1; - } - } - - int rc = execvp(path.c_str(), args.data()); - exit(rc); - } - else - { - int status; - waitpid(child_pid, &status, 0); - if(WIFEXITED(status)) - { - return WEXITSTATUS(status); - } - else - { - return -1; - } - } -} - -int exec_wait(const std::string& path, std::string& stdout, ...) -{ - va_list vl; - va_start(vl, stdout); - - std::vector args; - args.push_back(const_cast(path.c_str())); - - while(true) - { - const char* p = va_arg(vl, const char*); - if(p==NULL) break; - args.push_back(const_cast(p)); - } - va_end(vl); - - args.push_back(NULL); - - int pipefd[2]; - if (pipe(pipefd) == -1) - { - return -1; - } - - pid_t child_pid = fork(); - - if(child_pid==0) - { - environ = new char*[1]; - *environ=NULL; - - close(pipefd[0]); - - if(dup2(pipefd[1], 1)==-1) - { - return -1; - } - - if(dup2(pipefd[1], 2)==-1) - { - return -1; - } - - close(pipefd[1]); - - int rc = execvp(path.c_str(), args.data()); - exit(rc); - } - else - { - close(pipefd[1]); - - char buf[512]; - int r; - while( (r=read(pipefd[0], buf, 512))>0) - { - stdout.insert(stdout.end(), buf, buf+r); - } - - close(pipefd[0]); - - int status; - waitpid(child_pid, &status, 0); - if(WIFEXITED(status)) - { - return WEXITSTATUS(status); - } - else - { - return -1; - } - } -} - -std::string find_urbackupsrv_cmd() -{ - static std::string urbackupsrv_cmd; - - if(!urbackupsrv_cmd.empty()) - { - return urbackupsrv_cmd; - } - - if(exec_wait(BINDIR "/urbackupsrv", false, "--version", NULL)==1) - { - urbackupsrv_cmd = BINDIR "/urbackupsrv"; - return urbackupsrv_cmd; - } - else if(exec_wait("/usr/local/bin/urbackupsrv", false, "--version", NULL)==1) - { - urbackupsrv_cmd="/usr/local/bin/urbackupsrv"; - return urbackupsrv_cmd; - } - if(exec_wait("/sbin/urbackupsrv", false, "--version", NULL)==1) - { - urbackupsrv_cmd="/sbin/urbackupsrv"; - return urbackupsrv_cmd; - } - else if(exec_wait("/bin/urbackupsrv", false, "--version", NULL)==1) - { - urbackupsrv_cmd="/bin/urbackupsrv"; - return urbackupsrv_cmd; - } - else if(exec_wait("/usr/sbin/urbackupsrv", false, "--version", NULL)==1) - { - urbackupsrv_cmd="/usr/sbin/urbackupsrv"; - return urbackupsrv_cmd; - } - else if(exec_wait("/usr/bin/urbackupsrv", false, "--version", NULL)==1) - { - urbackupsrv_cmd="/usr/bin/urbackupsrv"; - return urbackupsrv_cmd; - } - else - { - urbackupsrv_cmd="urbackupsrv"; - return urbackupsrv_cmd; - } -} - - - -bool chown_dir(const std::string& dir) -{ - passwd* user_info = getpwnam("urbackup"); - if(user_info) - { - int rc = chown(dir.c_str(), user_info->pw_uid, user_info->pw_gid); - return rc!=-1; - } - return false; -} - -bool ubuntu_guestmount_fix() -{ -#ifdef __linux__ - //workaround for https://bugs.launchpad.net/ubuntu/+source/linux/+bug/759725 - if(getFile("/etc/os-release").find("NAME=\"Ubuntu\"")!=std::string::npos) - { - std::vector files = getFiles("/boot"); - for(size_t i=0;i=0) - { - mountpoint+=convert(partition); - } - - if(!os_directory_exists(mountpoint) - && !os_create_dir(mountpoint)) - { - std::cerr << "Error creating mountpoint at \"" << mountpoint << "\". Err: " << errno << std::endl; - close(loopd); - return false; - } - - chown_dir(mountpoint); - chown_dir("/dev/loop"+convert(devnum)); - - passwd* user_info = getpwnam("urbackup"); - std::string uid = "uid=0"; - std::string gid = "gid=0"; - if(user_info) - { - uid="uid="+convert(user_info->pw_uid); - gid="gid="+convert(user_info->pw_gid); - } - - ubuntu_guestmount_fix(); - - std::cout << "Guestmount..." << std::endl; - std::string stdout; - bool mount_ok=true; - if(exec_wait("guestmount", stdout, "-r", "-n", "--format=raw", "-a", ("/dev/loop"+convert(devnum)).c_str(), - "-o", "kernel_cache", - "-o", uid.c_str(), - "-o", gid.c_str(), - "-o", "allow_root", - "-m", "/dev/sda", - mountpoint.c_str(), NULL) ) - { - mount_ok=false; - std::cout << stdout; - size_t qpos = stdout.find("Did you mean to mount one of these filesystems"); - if(qpos!=std::string::npos) - { - stdout=stdout.substr(qpos); - stdout=getafter("guestmount:", stdout); - while(getbetween("(", ")", stdout)=="swap") - { - stdout=getafter("guestmount:", stdout); - } - stdout=trim(getuntil("(", stdout)); - std::cout << "Guestmount with dev " << stdout << std::endl; - - mount_ok=true; - if(exec_wait("guestmount", true, "-r", "-n", "--format=raw", "-a", ("/dev/loop"+convert(devnum)).c_str(), - "-o", "kernel_cache", - "-o", uid.c_str(), - "-o", gid.c_str(), - "-o", "allow_root", - "-m", stdout.c_str(), - mountpoint.c_str(), NULL) ) - { - mount_ok=false; - } - } - - if(!mount_ok) - { - close(loopd); - os_remove_dir(mountpoint); - return false; - } - } - - std::cout << stdout; - - close(loopd); - return true; -} -#else -bool mount_mdconfig(const std::string& imagepath, int partition) -{ - std::string mountpoint = ExtractFilePath(imagepath)+"_mnt"; - std::string unitpath = ExtractFilePath(imagepath)+"_unit"; - - if(partition>=0) - { - mountpoint+=convert(partition); - unitpath+=convert(partition); - } - - if(partition==-1) partition=0; - - if(os_directory_exists(mountpoint)) - { - std::cout << "unmounting..." << std::endl; - exec_wait(umount_path, true, mountpoint.c_str(), NULL); - - std::string unit = getFile(unitpath); - if(!unit.empty()) - { - std::cout << "Removing md" << unit << std::endl; - exec_wait(mdconfig_path, true, "-d", "-u", unit.c_str(), NULL); - } - unlink(unitpath.c_str()); - } - - if(!os_directory_exists(mountpoint) - && !os_create_dir(mountpoint)) - { - std::cerr << "Error creating mountpoint at \"" << mountpoint << "\". Err: " << errno << std::endl; - return false; - } - - chown_dir(mountpoint); - - for(size_t i=0;i<1024;++i) - { - int rc = exec_wait(mdconfig_path, true, "-a", "-t", "vnode", "-f", imagepath.c_str(), "-o", "readonly", "-u", convert(i).c_str(), NULL); - if(rc==0) - { - std::cout << "Found free md unit " << i << std::endl; - writestring(convert(i), unitpath.c_str()); - - std::cout << "Loading kernel module..." << std::endl; - exec_wait("/sbin/kldload", true, "fuse.ko", NULL); - - std::cout << "Mounting /dev/md"+convert(i)+"s"+convert(partition+1)<<" at " << mountpoint << " ..." << std::endl; - if(system(("ntfs-3g -o ro /dev/md"+convert(i)+"s"+convert(partition+1)+" \""+ mountpoint+"\"").c_str())==0) - { - return true; - } - else - { - std::cout << "Mounting failed. Removing md device..." << std::endl; - exec_wait(mdconfig_path, true, "-d", "-u", convert(i).c_str(), NULL); - unlink(unitpath.c_str()); - return false; - } - } - } - - std::cout << "No free mdconfig unit found." << std::endl; - - return false; -} -#endif - -bool mount_image(const std::string& imagepath, int partition, int64 offset, int64 length) -{ - std::string ext = findextension(imagepath); - - if(ext=="raw") - { -#ifdef __linux__ - return mount_linux_loop(imagepath, partition, offset, length); -#else - return mount_mdconfig(imagepath, partition); -#endif - } - else - { - std::string mountpoint = ExtractFilePath(imagepath)+"/contents"; - - if(partition>=0) - { - mountpoint+=convert(partition); - } - - if(os_directory_exists(mountpoint) || errno==EACCES || errno==ENOTCONN ) - { - exec_wait("guestunmount", true, mountpoint.c_str(), NULL); - } - - if(!os_directory_exists(mountpoint) - && !os_create_dir(mountpoint)) - { - std::cerr << "Error creating mountpoint at \"" << mountpoint << "\". Err: " << errno << std::endl; - return false; - } - - chown_dir(mountpoint); - - std::string devpoint = ExtractFilePath(imagepath)+"/device"; - - if(partition>=0) - { - devpoint+=convert(partition); - } - - if(os_directory_exists(devpoint) || errno==EACCES || errno==ENOTCONN ) - { - if(exec_wait("fusermount", true, "-u", devpoint.c_str(), NULL)) - { - exec_wait("fusermount", true, "-u", "-z", devpoint.c_str(), NULL); - } - } - - if(!os_directory_exists(devpoint) - && !os_create_dir(devpoint)) - { - std::cerr << "Error creating devpoint at \"" << devpoint << "\". Err: " << errno << std::endl; - os_remove_dir(mountpoint); - return false; - } - - chown_dir(devpoint); - - std::string mount_options=""; - passwd* user_info = getpwnam("urbackup"); - if(user_info) - { - mount_options+="uid="+convert(user_info->pw_uid)+",gid="+convert(user_info->pw_gid)+",allow_root"; - } - - ubuntu_guestmount_fix(); - - if(exec_wait(find_urbackupsrv_cmd(), true, "mount-vhd", "-f", imagepath.c_str(), "-m", mountpoint.c_str(), "-t", devpoint.c_str(), "-o", mount_options.c_str(), "--guestmount", NULL)) - { - std::cout << "UrBackup mount process returned non-zero return code" << std::endl; - os_remove_dir(mountpoint); - os_remove_dir(devpoint); - return false; - } - - return true; - } -} - -bool unmount_image(const std::string& imagepath, int partition) -{ - std::string ext = findextension(imagepath); - std::string mountpoint; - if(ext=="raw") - { - mountpoint = ExtractFilePath(imagepath)+"_mnt"; - } - else - { - mountpoint = ExtractFilePath(imagepath)+"/contents"; - } - - if(partition>=0) - { - mountpoint+=convert(partition); - } - - std::cout << "Mountpoint: " << mountpoint << std::endl; - - bool ret=true; - - if(os_directory_exists(mountpoint) || errno==EACCES || errno==ENOTCONN) - { -#ifdef __linux__ - std::cout << "Guestunmount..." << std::endl; - if(exec_wait("guestunmount", true, mountpoint.c_str(), NULL)) - { - std::cerr << "Unmounting \"" << mountpoint << "\" failed." << std::endl; - ret = false; - } -#else - if(exec_wait(umount_path, true, mountpoint.c_str(), NULL)) - { - exec_wait(umount_path, true, "-f", mountpoint.c_str(), NULL); - std::cerr << "Unmounting \"" << mountpoint << "\" failed." << std::endl; - ret = false; - } -#endif - - os_remove_dir(mountpoint); - } - -#ifndef __linux__ - std::string unitpath = ExtractFilePath(imagepath)+"_unit"; - if(partition>=0) - { - unitpath+=convert(partition); - } - std::string unit = getFile(unitpath); - if(!unit.empty()) - { - std::cout << "Removing md device " << unit << std::endl; - exec_wait(mdconfig_path, true, "-d", "-u", unit.c_str(), NULL); - } - unlink(unitpath.c_str()); -#endif - - std::string devpoint = ExtractFilePath(imagepath)+"/device"; - if(partition>=0) - { - devpoint+=convert(partition); - } - - if(ext!="raw" && (os_directory_exists(devpoint) || errno==EACCES || errno==ENOTCONN) ) - { - if(exec_wait("fusermount", true, "-u", devpoint.c_str(), NULL)) - { - exec_wait("fusermount", true, "-u", "-z", devpoint.c_str(), NULL); - exec_wait(umount_path, true, devpoint.c_str(), NULL); - } - - os_remove_dir(devpoint); - } - - return ret; -} - -int main(int argc, char *argv[]) -{ - if(argc<2) - { - std::cout << "Not enough parameters" << std::endl; - return 1; - } - - std::string backupfolder=getBackupfolderPath(); - - if(backupfolder.empty()) - { - std::cout << "Backupfolder not set" << std::endl; - return 1; - } - - if(seteuid(0)!=0) - { - std::cout << "Cannot become root user" << std::endl; - return 1; - } - - std::string cmd = argv[1]; - if(cmd=="mount") - { - if(argc<5) - { - std::cout << "Not enough parameters for mount" << std::endl; - return 1; - } - -#ifdef __FreeBSD__ - //system() on FreeBSD seems to not close file descriptors - for (int fd=3; fd<10000; fd++) close(fd); -#endif - std::string clientname=handleFilename(argv[2]); - std::string name=handleFilename(argv[3]); - std::string imagename=handleFilename(argv[4]); - - int partition=-1; - int64 offset=-1; - int64 length=0; - if(argc>7) - { - partition = watoi(argv[5]); - offset=watoi64(argv[6]); - length=watoi64(argv[7]); - } - - return mount_image(backupfolder + os_file_sep() + clientname + os_file_sep() + name + os_file_sep() + imagename, partition, offset, length)?0:1; - } - else if(cmd=="umount") - { - if(argc<5) - { - std::cout << "Not enough parameters for umount" << std::endl; - return 1; - } - - std::string clientname=handleFilename(argv[2]); - std::string name=handleFilename(argv[3]); - std::string imagename=handleFilename(argv[4]); - - int partition=-1; - - if(argc>5) - { - partition=watoi(argv[5]); - } - - return unmount_image(backupfolder + os_file_sep() + clientname + os_file_sep() + name + os_file_sep() + imagename, partition)?0:1; - } - else if(cmd=="test") - { -#ifndef WITH_FUSEPLUGIN - std::cerr << "TEST FAILED: Please compile with mountvhd (./configure --with-mountvhd)" << std::endl; - return 1; -#endif -#if defined(__linux__) - if(exec_wait("guestmount", false, "--version", NULL)!=0) - { - std::cerr << "TEST FAILED: guestmount is missing (libguestfs-tools)" << std::endl; - return 1; - } -#elif defined(__FreeBSD__) - if(exec_wait(mdconfig_path, false, "-l", NULL)!=0) - { - std::cerr << "TEST FAILED: mdconfig not present" << std::endl; - return 1; - } -#else - std::cerr << "TEST FAILED: Not FreeBSD or Linux" << std::endl; - return 1; -#endif - std::cout << "MOUNT TEST OK" << std::endl; - return 0; - } - else - { - std::cout << "Command not found" << std::endl; - return 1; - } -} - +#include +#include +#include +#include "../stringtools.h" +#include "../urbackupcommon/os_functions.h" +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#endif +#include +#include +extern char **environ; + +namespace +{ +std::vector get_child_env_store() +{ + std::vector env; + + const char* whitelist[] = { + "LIBGUESTFS_BACKEND", + "LIBGUESTFS_BACKEND_SETTINGS", + "LIBGUESTFS_PATH", + "LIBGUESTFS_CACHEDIR", + "LIBGUESTFS_TMPDIR", + "LIBGUESTFS_DEBUG", + "LIBGUESTFS_TRACE", + "TMPDIR", + NULL + }; + + for(size_t i=0; whitelist[i]!=NULL; ++i) + { + const char* val = getenv(whitelist[i]); + if(val!=NULL) + { + env.push_back(std::string(whitelist[i])+"="+val); + } + } + + return env; +} + +char** get_child_env(std::vector& env_store, std::vector& env_ptrs) +{ + env_ptrs.clear(); + + for(size_t i=0; i(env_store[i].c_str())); + } + + env_ptrs.push_back(NULL); + return env_ptrs.data(); +} + +bool elevate_child_to_root() +{ + if(geteuid()==0) + { + if(setgid(0)!=0) + { + return false; + } + + if(setuid(0)!=0) + { + return false; + } + } + + return true; +} +} + +#define DEF_Server +#include "../Server.h" +#include "../config.h" + +#ifndef LOOP_SET_DIRECT_IO +#define LOOP_SET_DIRECT_IO 0x4C08 +#endif +#ifndef LOOP_CTL_GET_FREE +#define LOOP_CTL_GET_FREE 0x4C82 +#endif +#ifndef O_CLOEXEC +#define O_CLOEXEC 0 +#endif +#define LOCAL_LO_FLAGS_AUTOCLEAR 4 + +IServer *Server; + +const char* mdconfig_path = "/sbin/mdconfig"; +const char* umount_path = "/sbin/umount"; + + +std::string getBackupfolderPath() +{ + std::string fn=trim(getFile("/etc/urbackup/backupfolder")); + if(fn.find("\n")!=std::string::npos) + fn=getuntil("\n", fn); + if(fn.find("\r")!=std::string::npos) + fn=getuntil("\r", fn); + + return fn; +} + +std::string handleFilename(std::string fn) +{ + fn=conv_filename(fn); + if(fn=="..") + { + return ""; + } + return fn; +} + +int exec_wait(const std::string& path, bool keep_stdout, ...) +{ + va_list vl; + va_start(vl, keep_stdout); + + std::vector args; + args.push_back(const_cast(path.c_str())); + + while(true) + { + const char* p = va_arg(vl, const char*); + if(p==NULL) break; + args.push_back(const_cast(p)); + } + va_end(vl); + + args.push_back(NULL); + + pid_t child_pid = fork(); + + if(child_pid==0) + { + std::vector env_store = get_child_env_store(); + std::vector env_ptrs; + environ = get_child_env(env_store, env_ptrs); + + if(!elevate_child_to_root()) + { + exit(126); + } + + if(!keep_stdout) + { + int nullfd = open("/dev/null", O_WRONLY); + + if(nullfd!=-1) + { + if(dup2(nullfd, 1)==-1) + { + return -1; + } + + if(dup2(nullfd, 2)==-1) + { + return -1; + } + } + else + { + return -1; + } + } + + int rc = execvp(path.c_str(), args.data()); + exit(rc); + } + else + { + int status; + waitpid(child_pid, &status, 0); + if(WIFEXITED(status)) + { + return WEXITSTATUS(status); + } + else + { + return -1; + } + } +} + +int exec_wait(const std::string& path, std::string& stdout, ...) +{ + va_list vl; + va_start(vl, stdout); + + std::vector args; + args.push_back(const_cast(path.c_str())); + + while(true) + { + const char* p = va_arg(vl, const char*); + if(p==NULL) break; + args.push_back(const_cast(p)); + } + va_end(vl); + + args.push_back(NULL); + + int pipefd[2]; + if (pipe(pipefd) == -1) + { + return -1; + } + + pid_t child_pid = fork(); + + if(child_pid==0) + { + std::vector env_store = get_child_env_store(); + std::vector env_ptrs; + environ = get_child_env(env_store, env_ptrs); + + if(!elevate_child_to_root()) + { + exit(126); + } + + close(pipefd[0]); + + if(dup2(pipefd[1], 1)==-1) + { + return -1; + } + + if(dup2(pipefd[1], 2)==-1) + { + return -1; + } + + close(pipefd[1]); + + int rc = execvp(path.c_str(), args.data()); + exit(rc); + } + else + { + close(pipefd[1]); + + char buf[512]; + int r; + while( (r=read(pipefd[0], buf, 512))>0) + { + stdout.insert(stdout.end(), buf, buf+r); + } + + close(pipefd[0]); + + int status; + waitpid(child_pid, &status, 0); + if(WIFEXITED(status)) + { + return WEXITSTATUS(status); + } + else + { + return -1; + } + } +} + + + +bool path_exists_lstat(const std::string& path) +{ + struct stat st; + return lstat(path.c_str(), &st)==0; +} + +#ifdef __linux__ +void cleanup_fuse_mountpoint(const std::string& path, bool guestmount) +{ + if(guestmount) + { + exec_wait("guestunmount", true, path.c_str(), NULL); + } + + exec_wait("fusermount", true, "-u", path.c_str(), NULL); + exec_wait("fusermount", true, "-u", "-z", path.c_str(), NULL); + exec_wait(umount_path, true, path.c_str(), NULL); + exec_wait(umount_path, true, "-l", path.c_str(), NULL); +} +#endif + +bool ensure_mount_dir(const std::string& path, const std::string& what) +{ + errno = 0; + if(os_directory_exists(path)) + { + return true; + } + + int exists_errno = errno; + +#ifdef __linux__ + if(exists_errno==EACCES || exists_errno==ENOTCONN || path_exists_lstat(path)) + { + cleanup_fuse_mountpoint(path, what=="mountpoint"); + + errno = 0; + if(os_directory_exists(path)) + { + return true; + } + + if(path_exists_lstat(path)) + { + if(rmdir(path.c_str())!=0 && errno!=ENOENT) + { + std::cerr << "Error removing stale " << what << " at \"" << path << "\". Err: " << errno << std::endl; + return false; + } + } + } +#endif + + errno = 0; + if(os_create_dir(path)) + { + return true; + } + + if(errno==EEXIST) + { + return os_directory_exists(path) || path_exists_lstat(path); + } + + std::cerr << "Error creating " << what << " at \"" << path << "\". Err: " << errno << std::endl; + return false; +} + +std::string find_urbackupsrv_cmd() +{ + static std::string urbackupsrv_cmd; + + if(!urbackupsrv_cmd.empty()) + { + return urbackupsrv_cmd; + } + + if(exec_wait(BINDIR "/urbackupsrv", false, "--version", NULL)==1) + { + urbackupsrv_cmd = BINDIR "/urbackupsrv"; + return urbackupsrv_cmd; + } + else if(exec_wait("/usr/local/bin/urbackupsrv", false, "--version", NULL)==1) + { + urbackupsrv_cmd="/usr/local/bin/urbackupsrv"; + return urbackupsrv_cmd; + } + if(exec_wait("/sbin/urbackupsrv", false, "--version", NULL)==1) + { + urbackupsrv_cmd="/sbin/urbackupsrv"; + return urbackupsrv_cmd; + } + else if(exec_wait("/bin/urbackupsrv", false, "--version", NULL)==1) + { + urbackupsrv_cmd="/bin/urbackupsrv"; + return urbackupsrv_cmd; + } + else if(exec_wait("/usr/sbin/urbackupsrv", false, "--version", NULL)==1) + { + urbackupsrv_cmd="/usr/sbin/urbackupsrv"; + return urbackupsrv_cmd; + } + else if(exec_wait("/usr/bin/urbackupsrv", false, "--version", NULL)==1) + { + urbackupsrv_cmd="/usr/bin/urbackupsrv"; + return urbackupsrv_cmd; + } + else + { + urbackupsrv_cmd="urbackupsrv"; + return urbackupsrv_cmd; + } +} + + + +bool chown_dir(const std::string& dir) +{ + passwd* user_info = getpwnam("urbackup"); + if(user_info) + { + int rc = chown(dir.c_str(), user_info->pw_uid, user_info->pw_gid); + return rc!=-1; + } + return false; +} + +bool ubuntu_guestmount_fix() +{ +#ifdef __linux__ + //workaround for https://bugs.launchpad.net/ubuntu/+source/linux/+bug/759725 + if(getFile("/etc/os-release").find("NAME=\"Ubuntu\"")!=std::string::npos) + { + std::vector files = getFiles("/boot"); + for(size_t i=0;i=0) + { + mountpoint+=convert(partition); + } + + if(!os_directory_exists(mountpoint) + && !os_create_dir(mountpoint)) + { + std::cerr << "Error creating mountpoint at \"" << mountpoint << "\". Err: " << errno << std::endl; + close(loopd); + return false; + } + + chown_dir(mountpoint); + chown_dir("/dev/loop"+convert(devnum)); + + passwd* user_info = getpwnam("urbackup"); + std::string uid = "uid=0"; + std::string gid = "gid=0"; + if(user_info) + { + uid="uid="+convert(user_info->pw_uid); + gid="gid="+convert(user_info->pw_gid); + } + + ubuntu_guestmount_fix(); + + std::cout << "Guestmount..." << std::endl; + std::string stdout; + bool mount_ok=true; + if(exec_wait("guestmount", stdout, "-r", "-n", "--format=raw", "-a", ("/dev/loop"+convert(devnum)).c_str(), + "-o", "kernel_cache", + "-o", uid.c_str(), + "-o", gid.c_str(), + "-o", "allow_root", + "-m", "/dev/sda", + mountpoint.c_str(), NULL) ) + { + mount_ok=false; + std::cout << stdout; + size_t qpos = stdout.find("Did you mean to mount one of these filesystems"); + if(qpos!=std::string::npos) + { + stdout=stdout.substr(qpos); + stdout=getafter("guestmount:", stdout); + while(getbetween("(", ")", stdout)=="swap") + { + stdout=getafter("guestmount:", stdout); + } + stdout=trim(getuntil("(", stdout)); + std::cout << "Guestmount with dev " << stdout << std::endl; + + mount_ok=true; + if(exec_wait("guestmount", true, "-r", "-n", "--format=raw", "-a", ("/dev/loop"+convert(devnum)).c_str(), + "-o", "kernel_cache", + "-o", uid.c_str(), + "-o", gid.c_str(), + "-o", "allow_root", + "-m", stdout.c_str(), + mountpoint.c_str(), NULL) ) + { + mount_ok=false; + } + } + + if(!mount_ok) + { + close(loopd); + os_remove_dir(mountpoint); + return false; + } + } + + std::cout << stdout; + + close(loopd); + return true; +} +#else +bool mount_mdconfig(const std::string& imagepath, int partition) +{ + std::string mountpoint = ExtractFilePath(imagepath)+"_mnt"; + std::string unitpath = ExtractFilePath(imagepath)+"_unit"; + + if(partition>=0) + { + mountpoint+=convert(partition); + unitpath+=convert(partition); + } + + if(partition==-1) partition=0; + + if(os_directory_exists(mountpoint)) + { + std::cout << "unmounting..." << std::endl; + exec_wait(umount_path, true, mountpoint.c_str(), NULL); + + std::string unit = getFile(unitpath); + if(!unit.empty()) + { + std::cout << "Removing md" << unit << std::endl; + exec_wait(mdconfig_path, true, "-d", "-u", unit.c_str(), NULL); + } + unlink(unitpath.c_str()); + } + + if(!os_directory_exists(mountpoint) + && !os_create_dir(mountpoint)) + { + std::cerr << "Error creating mountpoint at \"" << mountpoint << "\". Err: " << errno << std::endl; + return false; + } + + chown_dir(mountpoint); + + for(size_t i=0;i<1024;++i) + { + int rc = exec_wait(mdconfig_path, true, "-a", "-t", "vnode", "-f", imagepath.c_str(), "-o", "readonly", "-u", convert(i).c_str(), NULL); + if(rc==0) + { + std::cout << "Found free md unit " << i << std::endl; + writestring(convert(i), unitpath.c_str()); + + std::cout << "Loading kernel module..." << std::endl; + exec_wait("/sbin/kldload", true, "fuse.ko", NULL); + + std::cout << "Mounting /dev/md"+convert(i)+"s"+convert(partition+1)<<" at " << mountpoint << " ..." << std::endl; + if(system(("ntfs-3g -o ro /dev/md"+convert(i)+"s"+convert(partition+1)+" \""+ mountpoint+"\"").c_str())==0) + { + return true; + } + else + { + std::cout << "Mounting failed. Removing md device..." << std::endl; + exec_wait(mdconfig_path, true, "-d", "-u", convert(i).c_str(), NULL); + unlink(unitpath.c_str()); + return false; + } + } + } + + std::cout << "No free mdconfig unit found." << std::endl; + + return false; +} +#endif + +bool mount_image(const std::string& imagepath, int partition, int64 offset, int64 length) +{ + std::string ext = findextension(imagepath); + + if(ext=="raw") + { +#ifdef __linux__ + return mount_linux_loop(imagepath, partition, offset, length); +#else + return mount_mdconfig(imagepath, partition); +#endif + } + else + { + std::string mountpoint = ExtractFilePath(imagepath)+"/contents"; + + if(partition>=0) + { + mountpoint+=convert(partition); + } + + if(os_directory_exists(mountpoint) || errno==EACCES || errno==ENOTCONN ) + { + exec_wait("guestunmount", true, mountpoint.c_str(), NULL); + } + + if(!ensure_mount_dir(mountpoint, "mountpoint")) + { + return false; + } + + chown_dir(mountpoint); + + std::string devpoint = ExtractFilePath(imagepath)+"/device"; + + if(partition>=0) + { + devpoint+=convert(partition); + } + +#ifdef __linux__ + if(os_directory_exists(devpoint) || errno==EACCES || errno==ENOTCONN) + { + cleanup_fuse_mountpoint(devpoint, false); + os_remove_dir(devpoint); + } +#endif + + std::string mount_options=""; + passwd* user_info = getpwnam("urbackup"); + if(user_info) + { + mount_options+="uid="+convert(user_info->pw_uid)+",gid="+convert(user_info->pw_gid)+",allow_other"; + } + + ubuntu_guestmount_fix(); + + std::cerr << "Mounting VHD without guestmount device export" << std::endl; + std::string mount_stdout; + int mount_rc = exec_wait(find_urbackupsrv_cmd(), mount_stdout, "mount-vhd", "-f", imagepath.c_str(), "-m", mountpoint.c_str(), "-o", mount_options.c_str(), NULL); + std::cout << mount_stdout; + + if(mount_rc) + { + std::cout << "UrBackup mount process returned non-zero return code" << std::endl; +#ifdef __linux__ + cleanup_fuse_mountpoint(mountpoint, true); + cleanup_fuse_mountpoint(devpoint, false); +#endif + os_remove_dir(mountpoint); + os_remove_dir(devpoint); + return false; + } + + return true; + } +} + +bool unmount_image(const std::string& imagepath, int partition) +{ + std::string ext = findextension(imagepath); + std::string mountpoint; + if(ext=="raw") + { + mountpoint = ExtractFilePath(imagepath)+"_mnt"; + } + else + { + mountpoint = ExtractFilePath(imagepath)+"/contents"; + } + + if(partition>=0) + { + mountpoint+=convert(partition); + } + + std::cout << "Mountpoint: " << mountpoint << std::endl; + + bool ret=true; + + if(os_directory_exists(mountpoint) || errno==EACCES || errno==ENOTCONN) + { +#ifdef __linux__ + std::cout << "Guestunmount..." << std::endl; + if(exec_wait("guestunmount", true, mountpoint.c_str(), NULL)) + { + std::cerr << "Unmounting \"" << mountpoint << "\" failed." << std::endl; + ret = false; + } +#else + if(exec_wait(umount_path, true, mountpoint.c_str(), NULL)) + { + exec_wait(umount_path, true, "-f", mountpoint.c_str(), NULL); + std::cerr << "Unmounting \"" << mountpoint << "\" failed." << std::endl; + ret = false; + } +#endif + + os_remove_dir(mountpoint); + } + +#ifndef __linux__ + std::string unitpath = ExtractFilePath(imagepath)+"_unit"; + if(partition>=0) + { + unitpath+=convert(partition); + } + std::string unit = getFile(unitpath); + if(!unit.empty()) + { + std::cout << "Removing md device " << unit << std::endl; + exec_wait(mdconfig_path, true, "-d", "-u", unit.c_str(), NULL); + } + unlink(unitpath.c_str()); +#endif + + std::string devpoint = ExtractFilePath(imagepath)+"/device"; + if(partition>=0) + { + devpoint+=convert(partition); + } + + if(ext!="raw" && (os_directory_exists(devpoint) || errno==EACCES || errno==ENOTCONN) ) + { + if(exec_wait("fusermount", true, "-u", devpoint.c_str(), NULL)) + { + exec_wait("fusermount", true, "-u", "-z", devpoint.c_str(), NULL); + exec_wait(umount_path, true, devpoint.c_str(), NULL); + } + + os_remove_dir(devpoint); + } + + return ret; +} + +int main(int argc, char *argv[]) +{ + if(argc<2) + { + std::cout << "Not enough parameters" << std::endl; + return 1; + } + + std::string backupfolder=getBackupfolderPath(); + + if(backupfolder.empty()) + { + std::cout << "Backupfolder not set" << std::endl; + return 1; + } + + if(seteuid(0)!=0) + { + std::cout << "Cannot become root user" << std::endl; + return 1; + } + + std::string cmd = argv[1]; + if(cmd=="mount") + { + if(argc<5) + { + std::cout << "Not enough parameters for mount" << std::endl; + return 1; + } + +#ifdef __FreeBSD__ + //system() on FreeBSD seems to not close file descriptors + for (int fd=3; fd<10000; fd++) close(fd); +#endif + std::string clientname=handleFilename(argv[2]); + std::string name=handleFilename(argv[3]); + std::string imagename=handleFilename(argv[4]); + + int partition=-1; + int64 offset=-1; + int64 length=0; + if(argc>7) + { + partition = watoi(argv[5]); + offset=watoi64(argv[6]); + length=watoi64(argv[7]); + } + + return mount_image(backupfolder + os_file_sep() + clientname + os_file_sep() + name + os_file_sep() + imagename, partition, offset, length)?0:1; + } + else if(cmd=="umount") + { + if(argc<5) + { + std::cout << "Not enough parameters for umount" << std::endl; + return 1; + } + + std::string clientname=handleFilename(argv[2]); + std::string name=handleFilename(argv[3]); + std::string imagename=handleFilename(argv[4]); + + int partition=-1; + + if(argc>5) + { + partition=watoi(argv[5]); + } + + return unmount_image(backupfolder + os_file_sep() + clientname + os_file_sep() + name + os_file_sep() + imagename, partition)?0:1; + } + else if(cmd=="test") + { +#ifndef WITH_FUSEPLUGIN + std::cerr << "TEST FAILED: Please compile with mountvhd (./configure --with-mountvhd)" << std::endl; + return 1; +#endif +#if defined(__linux__) + if(exec_wait("guestmount", false, "--version", NULL)!=0) + { + std::cerr << "TEST FAILED: guestmount is missing (libguestfs-tools)" << std::endl; + return 1; + } +#elif defined(__FreeBSD__) + if(exec_wait(mdconfig_path, false, "-l", NULL)!=0) + { + std::cerr << "TEST FAILED: mdconfig not present" << std::endl; + return 1; + } +#else + std::cerr << "TEST FAILED: Not FreeBSD or Linux" << std::endl; + return 1; +#endif + std::cout << "MOUNT TEST OK" << std::endl; + return 0; + } + else + { + std::cout << "Command not found" << std::endl; + return 1; + } +} + diff --git a/org.urbackup.server.plist b/org.urbackup.server.plist new file mode 100644 index 000000000..d9eb12e01 --- /dev/null +++ b/org.urbackup.server.plist @@ -0,0 +1,50 @@ + + + + + + Disabled + + Label + org.urbackup.server + KeepAlive + + Crashed + + + + ProgramArguments + + /usr/local/bin/urbackupsrv + run + -l /var/log/urbackup/urbackup.log + + + WorkingDirectory /usr/local/var/urbackup + RunAtLoad + UserName urbackup + + + diff --git a/osx_installer/buildmacOSexclusions b/osx_installer/buildmacOSexclusions deleted file mode 100755 index 5aefbf364..000000000 --- a/osx_installer/buildmacOSexclusions +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash - -# DEBUGGING CONTROLS -# set -x -# trap read debug - - -#### Functions - -tidy_item() -{ - local trimmed1="$(echo -e "$1" | sed -e 's/[[:blank:]\",]*$//')" #Remove whitespace, the quote and comma at the end of each line - local trimmed2="$(echo -e "$trimmed1" | sed -e 's/^[[:blank:]\"]*//')" # Remove whitespace and quote at the start of the line - echo "$trimmed2" -} - - -#### Main - -force_rebuild=false -exclude_file="/Library/Application Support/UrBackup Client/var/urbackup/macos_exclusions.txt" - - -# Check if rebuild has been forced -if [ "$1" != "" ]; then - if [[ "$1" == "-f" ]] || [[ "$1" == "--force" ]]; then - force_rebuild=true - fi -fi - - -# Check macos_exclusion file doesn't already exist -if [ -f "$exclude_file" ]; then - if !($force_rebuild); then - exit 0 - else - rm "$exclude_file" - fi -fi - - -paths_excluded=$(defaults read "/System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist" PathsExcluded) -contents_excluded=$(defaults read "/System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist" ContentsExcluded) -file_contents_excluded=$(defaults read "/System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist" FileContentsExcluded) -user_paths_excluded=$(defaults read "/System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist" UserPathsExcluded) - - -paths_excluded_array=() - -IFS=$'\n' -for item in $paths_excluded; do - if [[ "$item" != "(" ]] && [[ "$item" != ")" ]]; then # Remove the leading and trailing paratheses - tidied="$(echo -e $(tidy_item $item))" - paths_excluded_array+=("$tidied") - fi -done - -for item in $contents_excluded; do - if [[ "$item" != "(" ]] && [[ "$item" != ")" ]]; then # Remove the leading and trailing paratheses - tidied="$(echo -e $(tidy_item $item))" - paths_excluded_array+=("$tidied/*") - fi -done - -# Need to refine this -for item in $file_contents_excluded; do - if [[ "$item" != "(" ]] && [[ "$item" != ")" ]]; then # Remove the leading and trailing paratheses - tidied="$(echo -e $(tidy_item $item))" - paths_excluded_array+=("$tidied/*") - fi -done - -for item in $user_paths_excluded; do - if [[ "$item" != "(" ]] && [[ "$item" != ")" ]]; then # Remove the leading and trailing paratheses - tidied="$(echo -e $(tidy_item $item))" - paths_excluded_array+=("/Users/*/$tidied") - fi -done - - -set -f -sorted_array=($(sort <<<"${paths_excluded_array[*]}")) - -mkdir -p "/Library/Application Support/UrBackup Client/var/urbackup" -touch $exclude_file - -for i in "${sorted_array[@]}"; do - echo "$i" >> $exclude_file -done - -exit 0 \ No newline at end of file diff --git a/osx_installer/info.plist b/osx_installer/info.plist index d80eeac8e..ec7ff026a 100644 --- a/osx_installer/info.plist +++ b/osx_installer/info.plist @@ -3,7 +3,7 @@ CFBundleGetInfoString - UrBackup Client Tray Icon GUI + UrBackup Client Tray Icon GUI $version_num_short$ ($git_rev$) CFBundleExecutable urbackupclientgui CFBundleIdentifier @@ -28,5 +28,7 @@ 1 LSMultipleInstancesProhibited + NSPrincipalClass + wxNSApplication - \ No newline at end of file + diff --git a/osx_installer/info_development.plist b/osx_installer/info_development.plist deleted file mode 100644 index b6d975519..000000000 --- a/osx_installer/info_development.plist +++ /dev/null @@ -1,32 +0,0 @@ - - - - - CFBundleGetInfoString - UrBackup Client Tray Icon GUI - CFBundleExecutable - urbackupclientgui - CFBundleIdentifier - org.urbackup.client.frontend - CFBundleName - UrBackup Client - CFBundleIconFile - urbackup.icns - CFBundleVersion - 0.1 - CFBundleShortVersionString - 0.1 - CFBundleInfoDictionaryVersion - 6.0 - CFBundlePackageType - APPL - IFMajorVersion - 0 - IFMinorVersion - 1 - LSUIElement - 1 - LSMultipleInstancesProhibited - - - \ No newline at end of file diff --git a/osx_installer/macOS_exclusion_overrides.txt b/osx_installer/macOS_exclusion_overrides.txt new file mode 100644 index 000000000..b05422347 --- /dev/null +++ b/osx_installer/macOS_exclusion_overrides.txt @@ -0,0 +1,21 @@ +# macos_exclusion_overrides.txt +# +# This file determines which of the standard macOS backup exclusions, as defined +# in Urbackup, are overridden - so as to force their inclusion in backups. +# +# This should not normally be required - and indeed, forcing the backup of some items +# may produce errors in the backup if the system does not allow access to them. +# +# The specific exclusion required to be overridden can be found in the Urbackup Client +# logfile - /Library/Logs/urbackup_client_backend.log +# Any item which is excluded due to a standard backup exclusion is noted against the +# exclusion term which prevents its backup. +# +# So, for example, to override the exclusion of the .Trash folder inside each user +# account, uncomment the following entry: +# +# /Users/:/.Trash +# +# Override entries should be exactly as quoted in the Urbackup Client logfile, and +# should be entered one-per-line without a preceding # + diff --git a/osx_installer/scripts2/postinstall b/osx_installer/scripts2/postinstall index 3389b7682..0e350def9 100755 --- a/osx_installer/scripts2/postinstall +++ b/osx_installer/scripts2/postinstall @@ -2,6 +2,12 @@ set -e +if test ! -f "/Library/Application Support/UrBackup Client/var/urbackup/macOS_exclusion_overrides.txt" +then + cp "$2/Contents/Resources/macOS_exclusion_overrides.txt" "/Library/Application Support/UrBackup Client/var/urbackup/macOS_exclusion_overrides.txt" + chmod 755 "/Library/Application Support/UrBackup Client/var/urbackup/macOS_exclusion_overrides.txt" +fi + if test -e "$1.cfg" then cp "$1.cfg" "/Library/Application Support/UrBackup Client/var/urbackup/initial_settings.cfg" @@ -15,18 +21,4 @@ fi /bin/launchctl load "/Library/LaunchDaemons/org.urbackup.client.plist" /bin/launchctl start "org.urbackup.client.backend" -if test ! -e "$1.silent" -then - #There is user interaction required here :( - "$2/Contents/MacOS/urbackupclientgui" register_login_item -else - for console_user in $(ps aux | grep "loginwindow" | grep -v grep | awk '{print $1;}') - do - if [ "$console_user" != "root" ] - then - sudo -u "$console_user" "$2/Contents/MacOS/urbackupclientgui" daemon - fi - done -fi -"$2/Contents/MacOS/bin/buildmacOSexclusions" diff --git a/pdb_dirs_client.txt b/pdb_dirs_client.txt index 1523064e4..a33bdf8d9 100644 --- a/pdb_dirs_client.txt +++ b/pdb_dirs_client.txt @@ -3,5 +3,7 @@ Release Release Server 2003 Release Service Release WinXP +ARM64 client\x64 -client\Release \ No newline at end of file +client\Release +client\ARM64 \ No newline at end of file diff --git a/post_install_osx_server.sh b/post_install_osx_server.sh new file mode 100755 index 000000000..9ffa775b0 --- /dev/null +++ b/post_install_osx_server.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env sh +# Run this after each "sudo make install" to verify permissions + +USERNAME=urbackup +GROUPNAME=urbackup +PLISTFILE=org.urbackup.server.plist + + +if [ $EUID -ne 0 ]; then + echo "This script must be run as root" + exit 1 +fi + +src_dir=$(dirname $0) + +# Create urbackup group +GROUPID=$(dscl . -read /Groups/$GROUPNAME PrimaryGroupID 2> /dev/null | awk '{print $2}') +if [ -z "$GROUPID" ]; then + echo "Could not find group $GROUPNAME. Please run ./preinstall_osx_server.sh before 'make install'." + exit 1 +else + echo "Found group '$GROUPNAME' with ID $GROUPID" +fi + +# Create urbackup user +USERID=$(dscl . -read /Users/$USERNAME UniqueID 2> /dev/null | awk '{print $2}') +if [ -z "$USERID" ]; then + echo "Could not find user $USERNAME. Please run ./preinstall_osx_server.sh before 'make install'." + exit 1 +else + echo "Found user '$USERNAME' with ID $USERID" +fi + +# Create log folder +if [ ! -d "/var/log/$USERNAME" ]; then + echo "Creating directory for log files /var/log/$USERNAME" + mkdir /var/log/$USERNAME +fi + +# Update permissions +echo "Updating permissions..." +chown $USERNAME:$GROUPNAME /var/log/$USERNAME +chown -R $USERNAME:$GROUPNAME /usr/local/var/urbackup + +if [ ! -f /usr/local/var/urbackup/backupfolder ]; then + echo "Could not find '/usr/local/var/urbackup/backupfolder' file. Please specify your backup folder in this file." + exit 1 +fi + +echo "Verifying permissions..." +BACKUPDIR=$(cat /usr/local/var/urbackup/backupfolder) +parts=$(echo $BACKUPDIR/ | awk 'BEGIN{FS="/"}{for (i=1; i < NF; i++) print $i}') + +unset path +for part in $parts +do + path="$path/$part" + printf " Testing access to $path... " + if ! sudo -u $USERNAME test -d $path ; then + echo "ERROR!" + echo "Could not walk into folder '$path' using user '$USERNAME'. Please verify permissions." + exit 1 + else + echo "OK" + fi +done + +# Generate unique name +TMPFILE=$BACKUPDIR/test_file.$$ + +# Try harder if a clash (a bit paranoid) +if [ -f $TMPFILE ]; then + TMPFILE=$TMPFILE.1 +fi + +printf "Testing file write permission in '$BACKUPDIR'... " + +if ! sudo -u $USERNAME touch "$TMPFILE" ; then +#if ! touch $TMPFILE ; then + echo "ERROR!" + echo "No write permission for folder '$BACKUPDIR' using user '$USERNAME'. Please verify permissions." + exit 1 +else + echo "OK" + rm $TMPFILE +fi + +echo "Verifying Launchd plist file..." + +if [ ! -f $src_dir/$PLISTFILE ]; then + echo "ERROR! Could not find launchd plist file: $src_dir/$PLISTFILE" + exit 1 +fi + +if [ -f /Library/LaunchDaemons/$PLISTFILE ]; then + echo "Launchd plist file found in '/Library/LaunchDaemons/$PLISTFILE'. Keeping it." +else + echo "Installing a template Launchd plist file in '/Library/LaunchDaemons/$PLISTFILE'..." + cp "$src_dir/$PLISTFILE" /Library/LaunchDaemons/ +fi + +echo "All steps done!" +echo +echo "General commands to be done:" +echo " Start Service: sudo launchctl load -w /Library/LaunchDaemons/$PLISTFILE" +echo " Stop Service: sudo launchctl unload -w /Library/LaunchDaemons/$PLISTFILE" +echo " Customize Launchd: sudo nano /Library/LaunchDaemons/$PLISTFILE" diff --git a/pre_install_osx_server.sh b/pre_install_osx_server.sh new file mode 100755 index 000000000..3d1b1f7b3 --- /dev/null +++ b/pre_install_osx_server.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env sh +# Run this before the first "sudo make install" to create 'urbackup' user in OS X + +USERNAME=urbackup +USERLONGNAME=urbackup.org +GROUPNAME=urbackup +GROUPLONGNAME=urbackup.org + + +if [ $EUID -ne 0 ]; then + echo "This script must be run as root" + exit 1 +fi + +getHiddenUserUid () { + local __UIDS=$(dscl . -list /Users UniqueID | awk '{print $2}' | sort -ugr) + + local __NewUID + for __NewUID in $__UIDS + do + if [[ $__NewUID -lt 499 ]] ; then + break; + fi + done + + echo $((__NewUID+1)) +} + +getGroupUid () { + local __UIDS=$(dscl . -list /Groups PrimaryGroupID | awk '{print $2}' | sort -ugr) + + local __NewUID + for __NewUID in $__UIDS + do + if [[ $__NewUID -lt 390 ]] ; then + break; + fi + done + + echo $((__NewUID+1)) +} + + +# Create urbackup group +GROUPID=$(dscl . -read /Groups/$GROUPNAME PrimaryGroupID 2> /dev/null | awk '{print $2}') +if [ -z "$GROUPID" ]; then + GROUPID=$(getGroupUid) + echo "Creating group '$GROUPNAME' with ID $GROUPID" + dseditgroup -i $GROUPID -r "$GROUPLONGNAME" -o create "$GROUPNAME" +else + echo "Found $GROUPNAME group with ID $GROUPID" +fi + +# Create urbackup user +USERID=$(dscl . -read /Users/$USERNAME UniqueID 2> /dev/null | awk '{print $2}') +if [ -z "$USERID" ]; then + USERID=$(getHiddenUserUid) + echo "Creating user '$USERNAME' with ID $USERID" + dscl . -create /Users/$USERNAME UniqueID "$USERID" + dscl . -append /Users/$USERNAME RealName "$USERLONGNAME" + dscl . -append /Users/$USERNAME PrimaryGroupID "$GROUPID" + dscl . -append /Users/$USERNAME NFSHomeDirectory "/usr/local/var/urbackup" + dscl . -append /Users/$USERNAME UserShell /usr/bin/false + dscl . -append /Users/$USERNAME IsHidden "1" + dseditgroup -o edit -t user -a $USERNAME $GROUPNAME + dseditgroup -o edit -t user -a $USERNAME daemon +else + echo "Found $USERNAME group with ID $USERID" +fi + +# Create log folder +if [ ! -d "/var/log/$USERNAME" ] +then + echo "Creating directory for log files /var/log/$USERNAME" + mkdir /var/log/$USERNAME +fi + +# Update permissions +echo "Updating permissions..." +chown $USERNAME:$GROUPNAME /var/log/$USERNAME + diff --git a/readme.md b/readme.md index 22950ab14..6d9ce5745 100644 --- a/readme.md +++ b/readme.md @@ -36,12 +36,12 @@ See the separate `readme-macos.md` for building instructions for macOS. ### Building on Windows -If git is in `PATH` you can download all dependencies by running `update_deps.bat`. +Build with Visual Studio 2019: -Afterwards opening and compiling the solution `UrBackupBackend.sln` with -Microsoft Visual Studio 2015 should work. + * Install [vcpkg](https://vcpkg.io/en/index.html) and run `vcpkg integrate install` + * Set global environment variable `VCPKG_FEATURE_FLAGS` to `manifests` + * Open `UrBackupBackend.sln` with Visual Studio 2019 and build (and run) `build_client.bat` and `build_server.bat` build the installers but you need to install a lot of dependencies like WiX, NSIS plus plugins, etc. -[![Build Status](https://travis-ci.org/uroni/urbackup_backend.svg?branch=dev)](https://travis-ci.org/uroni/urbackup_backend) diff --git a/restore_cd/restore_linux_img.sh b/restore_cd/restore_linux_img.sh index d60a27a2b..8625a6424 100644 --- a/restore_cd/restore_linux_img.sh +++ b/restore_cd/restore_linux_img.sh @@ -152,7 +152,7 @@ SERVER_CONNECTED=0 while [ $WTIME -lt 60 ] do sleep 1 - if $PREFIX/sbin/urbackuprestoreclientctl status 2>&1 | grep '"internet_connected": true' > /dev/null && ! $PREFIX/sbin/urbackuprestoreclientctl status 2>&1 | grep '"servers": []' + if $PREFIX/sbin/urbackuprestoreclientctl status 2>&1 | grep '"internet_connected": true' > /dev/null && ! $PREFIX/sbin/urbackuprestoreclientctl status 2>&1 | grep '"servers": \[\]' > /dev/null then echo "Successfully connected to Internet server. Logging in..." SERVER_CONNECTED=1 diff --git a/snapshot_helper/main.cpp b/snapshot_helper/main.cpp index e4b8d4a25..ee7d6910f 100644 --- a/snapshot_helper/main.cpp +++ b/snapshot_helper/main.cpp @@ -1,108 +1,109 @@ -#include -#include -#include +#include +#include +#include +#include #include -#include "../stringtools.h" -#include "../urbackupcommon/os_functions.h" -#include -#ifndef _WIN32 +#include "../stringtools.h" +#include "../urbackupcommon/os_functions.h" +#include +#ifndef _WIN32 #include #include #include -#include -#include +#include +#include #include -#include +#include extern char **environ; #endif #define DEF_Server -#include "../Server.h" - -const int mode_btrfs=0; -const int mode_zfs=1; +#include "../Server.h" + +const int mode_btrfs=0; +const int mode_zfs=1; const int mode_zfs_file=2; IServer *Server; - -#ifdef _WIN32 -#include - -bool CopyFolder(std::wstring src, std::wstring dst) -{ - if(!os_create_dir(dst)) - return false; - - std::vector curr_files=getFiles(src); - for(size_t i=0;i + +bool CopyFolder(std::wstring src, std::wstring dst) +{ + if(!os_create_dir(dst)) + return false; + + std::vector curr_files=getFiles(src); + for(size_t i=0;i args; - args.push_back(const_cast(path.c_str())); - - while(true) - { - const char* p = va_arg(vl, const char*); - if(p==NULL) break; - args.push_back(const_cast(p)); - } - va_end(vl); - - args.push_back(NULL); - - int pipefd[2]; - if (pipe(pipefd) == -1) - { - return -1; - } - - pid_t child_pid = fork(); - - if(child_pid==0) - { - environ = new char*[1]; - *environ=NULL; - - close(pipefd[0]); - - if(dup2(pipefd[1], 1)==-1) - { - return -1; - } - - close(pipefd[1]); - - int rc = execvp(path.c_str(), args.data()); - exit(rc); - } - else - { - close(pipefd[1]); - - char buf[512]; - int r; - while( (r=read(pipefd[0], buf, 512))>0) - { - stdout.insert(stdout.end(), buf, buf+r); - } - - close(pipefd[0]); - - int status; - waitpid(child_pid, &status, 0); - if(WIFEXITED(status)) - { - return WEXITSTATUS(status); - } - else - { - return -1; - } - } +} + +int exec_wait(const std::string& path, std::string& stdout, ...) +{ + va_list vl; + va_start(vl, stdout); + + std::vector args; + args.push_back(const_cast(path.c_str())); + + while(true) + { + const char* p = va_arg(vl, const char*); + if(p==NULL) break; + args.push_back(const_cast(p)); + } + va_end(vl); + + args.push_back(NULL); + + int pipefd[2]; + if (pipe(pipefd) == -1) + { + return -1; + } + + pid_t child_pid = fork(); + + if(child_pid==0) + { + environ = new char*[1]; + *environ=NULL; + + close(pipefd[0]); + + if(dup2(pipefd[1], 1)==-1) + { + return -1; + } + + close(pipefd[1]); + + int rc = execvp(path.c_str(), args.data()); + exit(rc); + } + else + { + close(pipefd[1]); + + char buf[512]; + int r; + while( (r=read(pipefd[0], buf, 512))>0) + { + stdout.insert(stdout.end(), buf, buf+r); + } + + close(pipefd[0]); + + int status; + waitpid(child_pid, &status, 0); + if(WIFEXITED(status)) + { + return WEXITSTATUS(status); + } + else + { + return -1; + } + } } bool chown_dir(const std::string& dir) @@ -248,404 +249,372 @@ bool chown_dir(const std::string& dir) return rc!=-1; } return false; -} - -std::string find_btrfs_cmd() -{ - static std::string btrfs_cmd; - - if(!btrfs_cmd.empty()) - { - return btrfs_cmd; - } - - if(exec_wait("btrfs", false, "--version", NULL)==0) - { - btrfs_cmd="btrfs"; - return btrfs_cmd; - } - else if(exec_wait("/sbin/btrfs", false, "--version", NULL)==0) - { - btrfs_cmd="/sbin/btrfs"; - return btrfs_cmd; - } - else if(exec_wait("/bin/btrfs", false, "--version", NULL)==0) - { - btrfs_cmd="/bin/btrfs"; - return btrfs_cmd; - } - else if(exec_wait("/usr/sbin/btrfs", false, "--version", NULL)==0) - { - btrfs_cmd="/usr/sbin/btrfs"; - return btrfs_cmd; - } - else if(exec_wait("/usr/bin/btrfs", false, "--version", NULL)==0) - { - btrfs_cmd="/usr/bin/btrfs"; - return btrfs_cmd; - } - else - { - btrfs_cmd="btrfs"; - return btrfs_cmd; - } -} - -std::string find_zfs_cmd() -{ - static std::string zfs_cmd; - - const char* const zfs_locations[] = { "zfs", - "/sbin/zfs", - "/bin/zfs", - "/usr/sbin/zfs", - "/usr/bin/zfs" }; - - for(size_t i=0;i& dependencies) -{ - zfs_elevate(); - - std::cout << "Searching for origin " << snapshot << std::endl; - - std::string snap_data; - int rc = exec_wait(find_zfs_cmd(), snap_data, "list", "-H", "-o", "name", NULL); - if(rc!=0) - return false; - - std::vector snaps; - Tokenize(snap_data, snaps, "\n"); - - std::string snap_folder = ExtractFilePath(snapshot); - for(size_t i=0;i&1 | grep \"ERROR: error accessing '-c'\"").c_str(), NULL); - if(compat_rc==0) - { - compat_rc=12; - } - } - - int rc; - if(compat_rc==12) - { - rc=exec_wait(find_btrfs_cmd(), !quiet, "subvolume", "delete", subvolume_folder.c_str(), NULL); - } - else - { - rc=exec_wait(find_btrfs_cmd(), !quiet, "subvolume", "delete", "-c", subvolume_folder.c_str(), NULL); + chown_dir(snapshot_dst); + return rc==0; + } + else if(mode==mode_zfs) + { + zfs_elevate(); + int rc=exec_wait(find_zfs_cmd(), true, "clone", (snapshot_src+"@ro").c_str(), snapshot_dst.c_str(), NULL); + chown_dir(snapshot_dst); + return rc==0; + } + return false; +#endif +} + +bool is_subvolume(int mode, std::string subvolume_folder) +{ +#ifdef _WIN32 + return true; +#else + if(mode==mode_btrfs) + { + int rc=exec_wait(find_btrfs_cmd(), false, "subvolume", "list", subvolume_folder.c_str(), NULL); + return rc==0; + } + else if(mode==mode_zfs) + { + zfs_elevate(); + int rc=exec_wait(find_zfs_cmd(), false, "list", subvolume_folder.c_str(), NULL); + return rc==0; + } + return false; +#endif +} + +bool remove_subvolume(int mode, std::string subvolume_folder, bool quiet=false) +{ +#ifdef _WIN32 + return os_remove_nonempty_dir(widen(subvolume_folder)); +#else + if(mode==mode_btrfs) + { + int compat_rc = exec_wait(find_btrfs_cmd(), false, "subvolume", "delete", "-c", NULL); + + if(compat_rc==1) + { + compat_rc = exec_wait("/bin/sh", false, "-c", (find_btrfs_cmd() + + " subvolume delete -c 2>&1 | grep \"ERROR: error accessing '-c'\"").c_str(), NULL); + if(compat_rc==0) + { + compat_rc=12; + } + } + + int rc; + if(compat_rc==12) + { + rc=exec_wait(find_btrfs_cmd(), !quiet, "subvolume", "delete", subvolume_folder.c_str(), NULL); + } + else + { + rc=exec_wait(find_btrfs_cmd(), !quiet, "subvolume", "delete", "-c", subvolume_folder.c_str(), NULL); + } + return rc==0; + } + else if(mode==mode_zfs) + { + zfs_elevate(); + int rc=exec_wait(find_zfs_cmd(), false, "destroy", "-r", subvolume_folder.c_str(), NULL); + + if (rc!=0) + { + std::cout << "Checking " << subvolume_folder << " for dependencies..." << std::endl; + std::string clone_data; + rc=exec_wait(find_zfs_cmd(), clone_data, "get", "-H", "-o", "value", "clones", (subvolume_folder+"@ro").c_str(), NULL); + std::string rename_name = ExtractFileName(subvolume_folder); + if(rc==0) + { + std::vector clones; + Tokenize(clone_data, clones, "\n"); + std::cout << "Dependencies exist..." << std::endl; + + if (exec_wait(find_zfs_cmd(), true, "rename", (subvolume_folder+"@ro").c_str(), (subvolume_folder+"@"+rename_name).c_str(), NULL)!=0 && is_subvolume(mode, subvolume_folder+"@ro")) + { + return false; + } + + for(size_t i=0;i dependencies; - if(!promote_dependencies(subvolume_folder+"@"+rename_name, dependencies)) - { - return false; - } - - rc = exec_wait(find_zfs_cmd(), true, "destroy", subvolume_folder.c_str(), NULL); - - if(rc==0) - { - for(size_t i=0;i= 3.6." << std::endl; suc=false; - } - else + } + else { if(getFile(clientdir+os_file_sep()+"B"+os_file_sep()+"test")!="test") { std::cout << "TEST FAILED: Cannot read reflinked file" << std::endl; suc=false; - } + } } } - - if(!remove_subvolume(mode_btrfs, clientdir+os_file_sep()+"A") ) - { - std::cout << "TEST FAILED: Removing subvolume A failed" << std::endl; + + if(!remove_subvolume(mode_btrfs, clientdir+os_file_sep()+"A") ) + { + std::cout << "TEST FAILED: Removing subvolume A failed" << std::endl; suc=false; - } - - if(!remove_subvolume(mode_btrfs, clientdir+os_file_sep()+"B") ) - { - std::cout << "TEST FAILED: Removing subvolume B failed" << std::endl; + } + + if(!remove_subvolume(mode_btrfs, clientdir+os_file_sep()+"B") ) + { + std::cout << "TEST FAILED: Removing subvolume B failed" << std::endl; suc=false; - } - - if(!os_remove_dir(clientdir)) - { - std::cout << "TEST FAILED: Removing test clientdir failed" << std::endl; - return 1; - } + } + + if(!os_remove_dir(clientdir)) + { + std::cout << "TEST FAILED: Removing test clientdir failed" << std::endl; + return 1; + } if(!suc) { return 1; } - } - else - { - std::cout << "TEST FAILED: Creating test clientdir \"" << clientdir << "\" failed" << std::endl; - - return zfs_test(); - } - std::cout << "BTRFS TEST OK" << std::endl; - return 10 + mode_btrfs; - } - else if(cmd=="issubvolume") - { - if(argc<5) - { - std::cout << "Not enough parameters for issubvolume" << std::endl; - return 1; - } - - std::string clientname=handleFilename(argv[3]); - std::string name=handleFilename(argv[4]); - - std::string subvolume_folder=backupfolder+os_file_sep()+clientname+os_file_sep()+name; - - return is_subvolume(mode, subvolume_folder)?0:1; - } - else if(cmd=="makereadonly") - { - if(argc<5) - { - std::cout << "Not enough parameters for makereadonly" << std::endl; - return 1; - } - - std::string clientname=handleFilename(argv[3]); - std::string name=handleFilename(argv[4]); - - std::string subvolume_folder=backupfolder+os_file_sep()+clientname+os_file_sep()+name; - - return make_readonly(mode, subvolume_folder)?0:1; - } - else - { - std::cout << "Command not found" << std::endl; - return 1; - } + } + else + { + std::cout << "TEST FAILED: Creating test clientdir \"" << clientdir << "\" failed" << std::endl; + + return zfs_test(); + } + std::cout << "BTRFS TEST OK" << std::endl; + return 10 + mode_btrfs; + } + else if(cmd=="issubvolume") + { + if(argc<5) + { + std::cout << "Not enough parameters for issubvolume" << std::endl; + return 1; + } + + std::string clientname=handleFilename(argv[3]); + std::string name=handleFilename(argv[4]); + + std::string subvolume_folder=backupfolder+os_file_sep()+clientname+os_file_sep()+name; + + return is_subvolume(mode, subvolume_folder)?0:1; + } + else if(cmd=="makereadonly") + { + if(argc<5) + { + std::cout << "Not enough parameters for makereadonly" << std::endl; + return 1; + } + + std::string clientname=handleFilename(argv[3]); + std::string name=handleFilename(argv[4]); + + std::string subvolume_folder=backupfolder+os_file_sep()+clientname+os_file_sep()+name; + + return make_readonly(mode, subvolume_folder)?0:1; + } + else + { + std::cout << "Command not found" << std::endl; + return 1; + } } diff --git a/sqlite/shell.c b/sqlite/shell.c index bbfe9cf52..07b9db46b 100644 --- a/sqlite/shell.c +++ b/sqlite/shell.c @@ -1,21 +1,45 @@ -/* DO NOT EDIT! -** This file is automatically generated by the script in the canonical -** SQLite source tree at tool/mkshellc.tcl. That script combines source -** code from various constituent source files of SQLite into this single -** "shell.c" file used to implement the SQLite command-line shell. -** -** Most of the code found below comes from the "src/shell.c.in" file in -** the canonical SQLite source tree. That main file contains "INCLUDE" -** lines that specify other files in the canonical source tree that are -** inserted to getnerate this complete program source file. +/* +** This is the amalgamated source code to the "sqlite3" or "sqlite3.exe" +** command-line shell (CLI) for SQLite. This file is automatically +** generated by the tool/mkshellc.tcl script from the following sources: ** -** The code from multiple files is combined into this single "shell.c" -** source file to help make the command-line program easier to compile. +** ext/expert/sqlite3expert.c +** ext/expert/sqlite3expert.h +** ext/intck/sqlite3intck.c +** ext/intck/sqlite3intck.h +** ext/misc/appendvfs.c +** ext/misc/base64.c +** ext/misc/base85.c +** ext/misc/completion.c +** ext/misc/decimal.c +** ext/misc/fileio.c +** ext/misc/ieee754.c +** ext/misc/memtrace.c +** ext/misc/pcachetrace.c +** ext/misc/regexp.c +** ext/misc/series.c +** ext/misc/sha1.c +** ext/misc/shathree.c +** ext/misc/sqlar.c +** ext/misc/sqlite3_stdio.c +** ext/misc/sqlite3_stdio.h +** ext/misc/stmtrand.c +** ext/misc/uint.c +** ext/misc/vfstrace.c +** ext/misc/windirent.h +** ext/misc/zipfile.c +** ext/qrf/qrf.c +** ext/qrf/qrf.h +** ext/recover/dbdata.c +** ext/recover/sqlite3recover.c +** ext/recover/sqlite3recover.h +** src/shell.c.in ** ** To modify this program, get a copy of the canonical SQLite source tree, -** edit the src/shell.c.in" and/or some of the other files that are included -** by "src/shell.c.in", then rerun the tool/mkshellc.tcl script. +** edit the src/shell.c.in file and/or some of the other files that are +** listed above, then rerun the command "make shell.c". */ +/************************* Begin src/shell.c.in ******************/ /* ** 2001 September 15 ** @@ -27,13 +51,56 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains code to implement the "sqlite" command line +** This file contains code to implement the "sqlite3" command line ** utility for accessing SQLite databases. */ #if (defined(_WIN32) || defined(WIN32)) && !defined(_CRT_SECURE_NO_WARNINGS) /* This needs to come before any includes for MSVC compiler */ #define _CRT_SECURE_NO_WARNINGS #endif +typedef unsigned int u32; +typedef unsigned short int u16; + +/* +** Limit input nesting via .read or any other input redirect. +** It's not too expensive, so a generous allowance can be made. +*/ +#define MAX_INPUT_NESTING 25 + +/* +** Used to prevent warnings about unused parameters +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** Number of elements in an array +*/ +#define ArraySize(X) (int)(sizeof(X)/sizeof(X[0])) + +/* +** Optionally #include a user-defined header, whereby compilation options +** may be set prior to where they take effect, but after platform setup. +** If SQLITE_CUSTOM_INCLUDE=? is defined, its value names the #include +** file. Note that this macro has a like effect on sqlite3.c compilation. +*/ +# define SHELL_STRINGIFY_(f) #f +# define SHELL_STRINGIFY(f) SHELL_STRINGIFY_(f) +#ifdef SQLITE_CUSTOM_INCLUDE +# include SHELL_STRINGIFY(SQLITE_CUSTOM_INCLUDE) +#endif + +/* +** If SQLITE_SHELL_FIDDLE is defined then the shell is modified +** somewhat for use as a WASM module in a web browser. This flag +** should only be used when building the "fiddle" web application, as +** the browser-mode build has much different user input requirements +** and this build mode rewires the user input subsystem to account for +** that. +*/ +#if defined(SQLITE_SHELL_FIDDLE) +# undef SQLITE_OMIT_LOAD_EXTENSION +# define SQLITE_OMIT_LOAD_EXTENSION 1 +#endif /* ** Warning pragmas copied from msvc.h in the core. @@ -74,23 +141,34 @@ # define _LARGEFILE_SOURCE 1 #endif +#if defined(SQLITE_SHELL_FIDDLE) && !defined(_POSIX_SOURCE) +/* +** emcc requires _POSIX_SOURCE (or one of several similar defines) +** to expose strdup(). +*/ +# define _POSIX_SOURCE +#endif + #include #include #include #include +#include +#include #include "sqlite3.h" typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; typedef unsigned char u8; -#if SQLITE_USER_AUTHENTICATION -# include "sqlite3userauth.h" -#endif #include #include +#ifndef _WIN32 +# include +# include +#endif #if !defined(_WIN32) && !defined(WIN32) # include -# if !defined(__RTP__) && !defined(_WRS_KERNEL) +# if !defined(__RTP__) && !defined(_WRS_KERNEL) && !defined(SQLITE_WASI) # include # endif #endif @@ -145,29 +223,35 @@ typedef unsigned char u8; # define SHELL_USE_LOCAL_GETLINE 1 #endif +#ifndef deliberate_fall_through +/* Quiet some compilers about some of our intentional code. */ +# if defined(GCC_VERSION) && GCC_VERSION>=7000000 +# define deliberate_fall_through __attribute__((fallthrough)); +# else +# define deliberate_fall_through +# endif +#endif #if defined(_WIN32) || defined(WIN32) -# include -# include -# define isatty(h) _isatty(h) -# ifndef access -# define access(f,m) _access((f),(m)) -# endif -# ifndef unlink -# define unlink _unlink -# endif -# ifndef strdup -# define strdup _strdup -# endif -# undef popen -# define popen _popen -# undef pclose -# define pclose _pclose +# include +# include +# define isatty(h) _isatty(h) +# ifndef access +# define access(f,m) _access((f),(m)) +# endif +# ifndef unlink +# define unlink _unlink +# endif +# ifndef strdup +# define strdup _strdup +# endif +# undef pclose +# define pclose _pclose #else /* Make sure isatty() has a prototype. */ extern int isatty(int); -# if !defined(__RTP__) && !defined(_WRS_KERNEL) +# if !defined(__RTP__) && !defined(_WRS_KERNEL) && !defined(SQLITE_WASI) /* popen and pclose are not C89 functions and so are ** sometimes omitted from the header */ extern FILE *popen(const char*,const char*); @@ -189,1849 +273,4103 @@ typedef unsigned char u8; #define IsSpace(X) isspace((unsigned char)X) #define IsDigit(X) isdigit((unsigned char)X) #define ToLower(X) (char)tolower((unsigned char)X) +#define IsAlnum(X) isalnum((unsigned char)X) +#define IsAlpha(X) isalpha((unsigned char)X) #if defined(_WIN32) || defined(WIN32) +#undef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN #include /* string conversion routines only needed on Win32 */ extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); -extern char *sqlite3_win32_mbcs_to_utf8_v2(const char *, int); -extern char *sqlite3_win32_utf8_to_mbcs_v2(const char *, int); extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); #endif -/* On Windows, we normally run with output mode of TEXT so that \n characters -** are automatically translated into \r\n. However, this behavior needs -** to be disabled in some cases (ex: when generating CSV output and when -** rendering quoted strings that contain \n characters). The following -** routines take care of that. +/************************* Begin ext/misc/sqlite3_stdio.h ******************/ +/* +** 2024-09-24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This header file contains definitions of interfaces that provide +** cross-platform I/O for UTF-8 content. +** +** On most platforms, the interfaces definitions in this file are +** just #defines. For example sqlite3_fopen() is a macro that resolves +** to the standard fopen() in the C-library. +** +** But Windows does not have a standard C-library, at least not one that +** can handle UTF-8. So for windows build, the interfaces resolve to new +** C-language routines contained in the separate sqlite3_stdio.c source file. +** +** So on all non-Windows platforms, simply #include this header file and +** use the interfaces defined herein. Then to run your application on Windows, +** also link in the accompanying sqlite3_stdio.c source file when compiling +** to get compatible interfaces. */ -#if defined(_WIN32) || defined(WIN32) -static void setBinaryMode(FILE *file, int isOutput){ - if( isOutput ) fflush(file); - _setmode(_fileno(file), _O_BINARY); -} -static void setTextMode(FILE *file, int isOutput){ - if( isOutput ) fflush(file); - _setmode(_fileno(file), _O_TEXT); -} +#ifndef _SQLITE3_STDIO_H_ +#define _SQLITE3_STDIO_H_ 1 +#ifdef _WIN32 +/**** Definitions For Windows ****/ +#include +#include +#include + +FILE *sqlite3_fopen(const char *zFilename, const char *zMode); +FILE *sqlite3_popen(const char *zCommand, const char *type); +char *sqlite3_fgets(char *s, int size, FILE *stream); +int sqlite3_fputs(const char *s, FILE *stream); +int sqlite3_fprintf(FILE *stream, const char *format, ...); +int sqlite3_vfprintf(FILE *stream, const char *format, va_list); +void sqlite3_fsetmode(FILE *stream, int mode); + + +#else +/**** Definitions For All Other Platforms ****/ +#include +#define sqlite3_fopen fopen +#define sqlite3_popen popen +#define sqlite3_fgets fgets +#define sqlite3_fputs fputs +#define sqlite3_fprintf fprintf +#define sqlite3_vfprintf vfprintf +#define sqlite3_fsetmode(F,X) /*no-op*/ + +#endif +#endif /* _SQLITE3_STDIO_H_ */ + +/************************* End ext/misc/sqlite3_stdio.h ********************/ +/************************* Begin ext/misc/sqlite3_stdio.c ******************/ +/* +** 2024-09-24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Implementation of standard I/O interfaces for UTF-8 that are missing +** on Windows. +*/ +#ifdef _WIN32 /* This file is a no-op on all platforms except Windows */ +#ifndef _SQLITE3_STDIO_H_ +/* #include "sqlite3_stdio.h" */ +#endif +#undef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#include +#include +#include +#include +#include +/* #include "sqlite3.h" */ +#include +#include +#include +#include + +/* +** If the SQLITE_U8TEXT_ONLY option is defined, then use O_U8TEXT +** when appropriate on all output. (Sometimes use O_BINARY when +** rendering ASCII text in cases where NL-to-CRLF expansion would +** not be correct.) +** +** If the SQLITE_U8TEXT_STDIO option is defined, then use O_U8TEXT +** when appropriate when writing to stdout or stderr. Use O_BINARY +** or O_TEXT (depending on things like the .mode and the .crlf setting +** in the CLI, or other context clues in other applications) for all +** other output channels. +** +** The default behavior, if neither of the above is defined is to +** use O_U8TEXT when writing to the Windows console (or anything +** else for which _isatty() returns true) and to use O_BINARY or O_TEXT +** for all other output channels. +** +** The SQLITE_USE_W32_FOR_CONSOLE_IO macro is also available. If +** defined, it forces the use of Win32 APIs for all console I/O, both +** input and output. This is necessary for some non-Microsoft run-times +** that implement stdio differently from Microsoft/Visual-Studio. +*/ +#if defined(SQLITE_U8TEXT_ONLY) +# define UseWtextForOutput(fd) 1 +# define UseWtextForInput(fd) 1 +# define IsConsole(fd) _isatty(_fileno(fd)) +#elif defined(SQLITE_U8TEXT_STDIO) +# define UseWtextForOutput(fd) ((fd)==stdout || (fd)==stderr) +# define UseWtextForInput(fd) ((fd)==stdin) +# define IsConsole(fd) _isatty(_fileno(fd)) #else -# define setBinaryMode(X,Y) -# define setTextMode(X,Y) +# define UseWtextForOutput(fd) _isatty(_fileno(fd)) +# define UseWtextForInput(fd) _isatty(_fileno(fd)) +# define IsConsole(fd) 1 #endif +/* +** Global variables determine if simulated O_BINARY mode is to be +** used for stdout or other, respectively. Simulated O_BINARY mode +** means the mode is usually O_BINARY, but switches to O_U8TEXT for +** unicode characters U+0080 or greater (any character that has a +** multi-byte representation in UTF-8). This is the only way we +** have found to render Unicode characters on a Windows console while +** at the same time avoiding undesirable \n to \r\n translation. +*/ +static int simBinaryStdout = 0; +static int simBinaryOther = 0; -/* True if the timer is enabled */ -static int enableTimer = 0; -/* Return the current wall-clock time */ -static sqlite3_int64 timeOfDay(void){ - static sqlite3_vfs *clockVfs = 0; - sqlite3_int64 t; - if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); - if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ - clockVfs->xCurrentTimeInt64(clockVfs, &t); +/* +** Determine if simulated binary mode should be used for output to fd +*/ +static int UseBinaryWText(FILE *fd){ + if( fd==stdout || fd==stderr ){ + return simBinaryStdout; }else{ - double r; - clockVfs->xCurrentTime(clockVfs, &r); - t = (sqlite3_int64)(r*86400000.0); + return simBinaryOther; } - return t; } -#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) -#include -#include -/* VxWorks does not support getrusage() as far as we can determine */ -#if defined(_WRS_KERNEL) || defined(__RTP__) -struct rusage { - struct timeval ru_utime; /* user CPU time used */ - struct timeval ru_stime; /* system CPU time used */ -}; -#define getrusage(A,B) memset(B,0,sizeof(*B)) -#endif +/* +** Work-alike for the fopen() routine from the standard C library. +*/ +FILE *sqlite3_fopen(const char *zFilename, const char *zMode){ + FILE *fp = 0; + wchar_t *b1, *b2; + int sz1, sz2; + + sz1 = (int)strlen(zFilename); + sz2 = (int)strlen(zMode); + b1 = sqlite3_malloc64( (sz1+1)*sizeof(b1[0]) ); + b2 = sqlite3_malloc64( (sz2+1)*sizeof(b1[0]) ); + if( b1 && b2 ){ + sz1 = MultiByteToWideChar(CP_UTF8, 0, zFilename, sz1, b1, sz1); + b1[sz1] = 0; + sz2 = MultiByteToWideChar(CP_UTF8, 0, zMode, sz2, b2, sz2); + b2[sz2] = 0; + fp = _wfopen(b1, b2); + } + sqlite3_free(b1); + sqlite3_free(b2); + simBinaryOther = 0; + return fp; +} -/* Saved resource information for the beginning of an operation */ -static struct rusage sBegin; /* CPU time at start */ -static sqlite3_int64 iBegin; /* Wall-clock time at start */ /* -** Begin timing an operation +** Work-alike for the popen() routine from the standard C library. */ -static void beginTimer(void){ - if( enableTimer ){ - getrusage(RUSAGE_SELF, &sBegin); - iBegin = timeOfDay(); +FILE *sqlite3_popen(const char *zCommand, const char *zMode){ + FILE *fp = 0; + wchar_t *b1, *b2; + int sz1, sz2; + + sz1 = (int)strlen(zCommand); + sz2 = (int)strlen(zMode); + b1 = sqlite3_malloc64( (sz1+1)*sizeof(b1[0]) ); + b2 = sqlite3_malloc64( (sz2+1)*sizeof(b1[0]) ); + if( b1 && b2 ){ + sz1 = MultiByteToWideChar(CP_UTF8, 0, zCommand, sz1, b1, sz1); + b1[sz1] = 0; + sz2 = MultiByteToWideChar(CP_UTF8, 0, zMode, sz2, b2, sz2); + b2[sz2] = 0; + fp = _wpopen(b1, b2); } + sqlite3_free(b1); + sqlite3_free(b2); + return fp; } -/* Return the difference of two time_structs in seconds */ -static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ - return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + - (double)(pEnd->tv_sec - pStart->tv_sec); +/* +** Work-alike for fgets() from the standard C library. +*/ +char *sqlite3_fgets(char *buf, int sz, FILE *in){ + if( UseWtextForInput(in) ){ + /* When reading from the command-prompt in Windows, it is necessary + ** to use _O_WTEXT input mode to read UTF-16 characters, then translate + ** that into UTF-8. Otherwise, non-ASCII characters all get translated + ** into '?'. + */ + wchar_t *b1 = sqlite3_malloc64( sz*sizeof(wchar_t) ); + if( b1==0 ) return 0; +#ifdef SQLITE_USE_W32_FOR_CONSOLE_IO + DWORD nRead = 0; + if( IsConsole(in) + && ReadConsoleW(GetStdHandle(STD_INPUT_HANDLE), b1, sz-1, &nRead, 0) + ){ + b1[nRead] = 0; + }else +#endif + { + _setmode(_fileno(in), IsConsole(in) ? _O_WTEXT : _O_U8TEXT); + if( fgetws(b1, sz/4, in)==0 ){ + sqlite3_free(b1); + return 0; + } + } + WideCharToMultiByte(CP_UTF8, 0, b1, -1, buf, sz, 0, 0); + sqlite3_free(b1); + return buf; + }else{ + /* Reading from a file or other input source, just read bytes without + ** any translation. */ + return fgets(buf, sz, in); + } } /* -** Print the timing results. +** Send ASCII text as O_BINARY. But for Unicode characters U+0080 and +** greater, switch to O_U8TEXT. */ -static void endTimer(void){ - if( enableTimer ){ - sqlite3_int64 iEnd = timeOfDay(); - struct rusage sEnd; - getrusage(RUSAGE_SELF, &sEnd); - printf("Run Time: real %.3f user %f sys %f\n", - (iEnd - iBegin)*0.001, - timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), - timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); +static void piecemealOutput(wchar_t *b1, int sz, FILE *out){ + int i; + wchar_t c; + while( sz>0 ){ + for(i=0; i=0x80; i++){} + if( i>0 ){ + c = b1[i]; + b1[i] = 0; + fflush(out); + _setmode(_fileno(out), _O_U8TEXT); + fputws(b1, out); + fflush(out); + b1 += i; + b1[0] = c; + sz -= i; + }else{ + fflush(out); + _setmode(_fileno(out), _O_TEXT); + _setmode(_fileno(out), _O_BINARY); + fwrite(&b1[0], 1, 1, out); + for(i=1; i +/* #include "sqlite3.h" */ /* -** Used to prevent warnings about unused parameters -*/ -#define UNUSED_PARAMETER(x) (void)(x) +** Specification used by clients to define the output format they want +*/ +typedef struct sqlite3_qrf_spec sqlite3_qrf_spec; +struct sqlite3_qrf_spec { + unsigned char iVersion; /* Version number of this structure */ + unsigned char eStyle; /* Formatting style. "box", "csv", etc... */ + unsigned char eEsc; /* How to escape control characters in text */ + unsigned char eText; /* Quoting style for text */ + unsigned char eTitle; /* Quating style for the text of column names */ + unsigned char eBlob; /* Quoting style for BLOBs */ + unsigned char bTitles; /* True to show column names */ + unsigned char bWordWrap; /* Try to wrap on word boundaries */ + unsigned char bTextJsonb; /* Render JSONB blobs as JSON text */ + unsigned char eDfltAlign; /* Default alignment, no covered by aAlignment */ + unsigned char eTitleAlign; /* Alignment for column headers */ + unsigned char bSplitColumn; /* Wrap single-column output into many columns */ + unsigned char bBorder; /* Show outer border in Box and Table styles */ + short int nWrap; /* Wrap columns wider than this */ + short int nScreenWidth; /* Maximum overall table width */ + short int nLineLimit; /* Maximum number of lines for any row */ + short int nTitleLimit; /* Maximum number of characters in a title */ + unsigned int nMultiInsert; /* Add rows to one INSERT until size exceeds */ + int nCharLimit; /* Maximum number of characters in a cell */ + int nWidth; /* Number of entries in aWidth[] */ + int nAlign; /* Number of entries in aAlignment[] */ + short int *aWidth; /* Column widths */ + unsigned char *aAlign; /* Column alignments */ + char *zColumnSep; /* Alternative column separator */ + char *zRowSep; /* Alternative row separator */ + char *zTableName; /* Output table name */ + char *zNull; /* Rendering of NULL */ + char *(*xRender)(void*,sqlite3_value*); /* Render a value */ + int (*xWrite)(void*,const char*,sqlite3_int64); /* Write output */ + void *pRenderArg; /* First argument to the xRender callback */ + void *pWriteArg; /* First argument to the xWrite callback */ + char **pzOutput; /* Storage location for output string */ + /* Additional fields may be added in the future */ +}; /* -** Number of elements in an array +** Interfaces */ -#define ArraySize(X) (int)(sizeof(X)/sizeof(X[0])) +int sqlite3_format_query_result( + sqlite3_stmt *pStmt, /* SQL statement to run */ + const sqlite3_qrf_spec *pSpec, /* Result format specification */ + char **pzErr /* OUT: Write error message here */ +); /* -** If the following flag is set, then command execution stops -** at an error if we are not interactive. +** Range of values for sqlite3_qrf_spec.aWidth[] entries and for +** sqlite3_qrf_spec.mxColWidth and .nScreenWidth */ -static int bail_on_error = 0; +#define QRF_MAX_WIDTH 10000 +#define QRF_MIN_WIDTH 0 /* -** Threat stdin as an interactive input if the following variable -** is true. Otherwise, assume stdin is connected to a file or pipe. +** Output styles: */ -static int stdin_is_interactive = 1; +#define QRF_STYLE_Auto 0 /* Choose a style automatically */ +#define QRF_STYLE_Box 1 /* Unicode box-drawing characters */ +#define QRF_STYLE_Column 2 /* One record per line in neat columns */ +#define QRF_STYLE_Count 3 /* Output only a count of the rows of output */ +#define QRF_STYLE_Csv 4 /* Comma-separated-value */ +#define QRF_STYLE_Eqp 5 /* Format EXPLAIN QUERY PLAN output */ +#define QRF_STYLE_Explain 6 /* EXPLAIN output */ +#define QRF_STYLE_Html 7 /* Generate an XHTML table */ +#define QRF_STYLE_Insert 8 /* Generate SQL "insert" statements */ +#define QRF_STYLE_Json 9 /* Output is a list of JSON objects */ +#define QRF_STYLE_JObject 10 /* Independent JSON objects for each row */ +#define QRF_STYLE_Line 11 /* One column per line. */ +#define QRF_STYLE_List 12 /* One record per line with a separator */ +#define QRF_STYLE_Markdown 13 /* Markdown formatting */ +#define QRF_STYLE_Off 14 /* No query output shown */ +#define QRF_STYLE_Quote 15 /* SQL-quoted, comma-separated */ +#define QRF_STYLE_Stats 16 /* EQP-like output but with performance stats */ +#define QRF_STYLE_StatsEst 17 /* EQP-like output with planner estimates */ +#define QRF_STYLE_StatsVm 18 /* EXPLAIN-like output with performance stats */ +#define QRF_STYLE_Table 19 /* MySQL-style table formatting */ /* -** On Windows systems we have to know if standard output is a console -** in order to translate UTF-8 into MBCS. The following variable is -** true if translation is required. +** Quoting styles for text. +** Allowed values for sqlite3_qrf_spec.eText */ -static int stdout_is_console = 1; +#define QRF_TEXT_Auto 0 /* Choose text encoding automatically */ +#define QRF_TEXT_Plain 1 /* Literal text */ +#define QRF_TEXT_Sql 2 /* Quote as an SQL literal */ +#define QRF_TEXT_Csv 3 /* CSV-style quoting */ +#define QRF_TEXT_Html 4 /* HTML-style quoting */ +#define QRF_TEXT_Tcl 5 /* C/Tcl quoting */ +#define QRF_TEXT_Json 6 /* JSON quoting */ +#define QRF_TEXT_Relaxed 7 /* Relaxed SQL quoting */ /* -** The following is the open SQLite database. We make a pointer -** to this database a static variable so that it can be accessed -** by the SIGINT handler to interrupt database processing. +** Quoting styles for BLOBs +** Allowed values for sqlite3_qrf_spec.eBlob */ -static sqlite3 *globalDb = 0; +#define QRF_BLOB_Auto 0 /* Determine BLOB quoting using eText */ +#define QRF_BLOB_Text 1 /* Display content exactly as it is */ +#define QRF_BLOB_Sql 2 /* Quote as an SQL literal */ +#define QRF_BLOB_Hex 3 /* Hexadecimal representation */ +#define QRF_BLOB_Tcl 4 /* "\000" notation */ +#define QRF_BLOB_Json 5 /* A JSON string */ +#define QRF_BLOB_Size 6 /* Display the blob size only */ /* -** True if an interrupt (Control-C) has been received. +** Control-character escape modes. +** Allowed values for sqlite3_qrf_spec.eEsc */ -static volatile int seenInterrupt = 0; +#define QRF_ESC_Auto 0 /* Choose the ctrl-char escape automatically */ +#define QRF_ESC_Off 1 /* Do not escape control characters */ +#define QRF_ESC_Ascii 2 /* Unix-style escapes. Ex: U+0007 shows ^G */ +#define QRF_ESC_Symbol 3 /* Unicode escapes. Ex: U+0007 shows U+2407 */ /* -** This is the name of our program. It is set in main(), used -** in a number of other places, mostly for error messages. +** Allowed values for "boolean" fields, such as "bColumnNames", "bWordWrap", +** and "bTextJsonb". There is an extra "auto" variants so these are actually +** tri-state settings, not booleans. */ -static char *Argv0; +#define QRF_SW_Auto 0 /* Let QRF choose the best value */ +#define QRF_SW_Off 1 /* This setting is forced off */ +#define QRF_SW_On 2 /* This setting is forced on */ +#define QRF_Auto 0 /* Alternate spelling for QRF_*_Auto */ +#define QRF_No 1 /* Alternate spelling for QRF_SW_Off */ +#define QRF_Yes 2 /* Alternate spelling for QRF_SW_On */ /* -** Prompt strings. Initialized in main. Settable with -** .prompt main continue +** Possible alignment values alignment settings +** +** Horizontal Vertial +** ---------- -------- */ +#define QRF_ALIGN_Auto 0 /* auto auto */ +#define QRF_ALIGN_Left 1 /* left auto */ +#define QRF_ALIGN_Center 2 /* center auto */ +#define QRF_ALIGN_Right 3 /* right auto */ +#define QRF_ALIGN_Top 4 /* auto top */ +#define QRF_ALIGN_NW 5 /* left top */ +#define QRF_ALIGN_N 6 /* center top */ +#define QRF_ALIGN_NE 7 /* right top */ +#define QRF_ALIGN_Middle 8 /* auto middle */ +#define QRF_ALIGN_W 9 /* left middle */ +#define QRF_ALIGN_C 10 /* center middle */ +#define QRF_ALIGN_E 11 /* right middle */ +#define QRF_ALIGN_Bottom 12 /* auto bottom */ +#define QRF_ALIGN_SW 13 /* left bottom */ +#define QRF_ALIGN_S 14 /* center bottom */ +#define QRF_ALIGN_SE 15 /* right bottom */ +#define QRF_ALIGN_HMASK 3 /* Horizontal alignment mask */ +#define QRF_ALIGN_VMASK 12 /* Vertical alignment mask */ + +/* +** Auxiliary routines contined within this module that might be useful +** in other contexts, and which are therefore exported. +*/ +/* +** Return an estimate of the width, in columns, for the single Unicode +** character c. For normal characters, the answer is always 1. But the +** estimate might be 0 or 2 for zero-width and double-width characters. +** +** Different devices display unicode using different widths. So +** it is impossible to know that true display width with 100% accuracy. +** Inaccuracies in the width estimates might cause columns to be misaligned. +** Unfortunately, there is nothing we can do about that. */ -static char mainPrompt[20]; /* First line prompt. default: "sqlite> "*/ -static char continuePrompt[20]; /* Continuation prompt. default: " ...> " */ +int sqlite3_qrf_wcwidth(int c); /* -** Render output like fprintf(). Except, if the output is going to the -** console and if this is running on a Windows machine, translate the -** output from UTF-8 into MBCS. +** Return an estimate of the number of display columns used by the +** string in the argument. The width of individual characters is +** determined as for sqlite3_qrf_wcwidth(). VT100 escape code sequences +** are assigned a width of zero. */ -#if defined(_WIN32) || defined(WIN32) -void utf8_printf(FILE *out, const char *zFormat, ...){ - va_list ap; - va_start(ap, zFormat); - if( stdout_is_console && (out==stdout || out==stderr) ){ - char *z1 = sqlite3_vmprintf(zFormat, ap); - char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); - sqlite3_free(z1); - fputs(z2, out); - sqlite3_free(z2); - }else{ - vfprintf(out, zFormat, ap); - } - va_end(ap); +size_t sqlite3_qrf_wcswidth(const char*); + + +#ifdef __cplusplus } -#elif !defined(utf8_printf) -# define utf8_printf fprintf #endif +#endif /* !defined(SQLITE_QRF_H) */ +/************************* End ext/qrf/qrf.h ********************/ +/************************* Begin ext/qrf/qrf.c ******************/ /* -** Render output like fprintf(). This should not be used on anything that -** includes string formatting (e.g. "%s"). +** 2025-10-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Implementation of the Query Result-Format or "qrf" utility library for +** SQLite. See the README.md documentation for additional information. */ -#if !defined(raw_printf) -# define raw_printf fprintf +#ifndef SQLITE_QRF_H +#include "qrf.h" #endif +#include +#include +#include -/* Indicate out-of-memory and exit. */ -static void shell_out_of_memory(void){ - raw_printf(stderr,"Error: out of memory\n"); - exit(1); +#ifndef SQLITE_AMALGAMATION +/* typedef sqlite3_int64 i64; */ +#endif + +/* A single line in the EQP output */ +typedef struct qrfEQPGraphRow qrfEQPGraphRow; +struct qrfEQPGraphRow { + int iEqpId; /* ID for this row */ + int iParentId; /* ID of the parent row */ + qrfEQPGraphRow *pNext; /* Next row in sequence */ + char zText[1]; /* Text to display for this row */ +}; + +/* All EQP output is collected into an instance of the following */ +typedef struct qrfEQPGraph qrfEQPGraph; +struct qrfEQPGraph { + qrfEQPGraphRow *pRow; /* Linked list of all rows of the EQP output */ + qrfEQPGraphRow *pLast; /* Last element of the pRow list */ + int nWidth; /* Width of the graph */ + char zPrefix[400]; /* Graph prefix */ +}; + +/* +** Private state information. Subject to change from one release to the +** next. +*/ +typedef struct Qrf Qrf; +struct Qrf { + sqlite3_stmt *pStmt; /* The statement whose output is to be rendered */ + sqlite3 *db; /* The corresponding database connection */ + sqlite3_stmt *pJTrans; /* JSONB to JSON translator statement */ + char **pzErr; /* Write error message here, if not NULL */ + sqlite3_str *pOut; /* Accumulated output */ + int iErr; /* Error code */ + int nCol; /* Number of output columns */ + int expMode; /* Original sqlite3_stmt_isexplain() plus 1 */ + int mxWidth; /* Screen width */ + int mxHeight; /* nLineLimit */ + union { + struct { /* Content for QRF_STYLE_Line */ + int mxColWth; /* Maximum display width of any column */ + char **azCol; /* Names of output columns (MODE_Line) */ + } sLine; + qrfEQPGraph *pGraph; /* EQP graph (Eqp, Stats, and StatsEst) */ + struct { /* Content for QRF_STYLE_Explain */ + int nIndent; /* Slots allocated for aiIndent */ + int iIndent; /* Current slot */ + int *aiIndent; /* Indentation for each opcode */ + } sExpln; + unsigned int nIns; /* Bytes used for current INSERT stmt */ + } u; + sqlite3_int64 nRow; /* Number of rows handled so far */ + int *actualWidth; /* Actual width of each column */ + sqlite3_qrf_spec spec; /* Copy of the original spec */ +}; + +/* +** Data for substitute ctype.h functions. Used for x-platform +** consistency and so that '_' is counted as an alphabetic +** character. +** +** 0x01 - space +** 0x02 - digit +** 0x04 - alphabetic, including '_' +*/ +static const char qrfCType[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; +#define qrfSpace(x) ((qrfCType[(unsigned char)x]&1)!=0) +#define qrfDigit(x) ((qrfCType[(unsigned char)x]&2)!=0) +#define qrfAlpha(x) ((qrfCType[(unsigned char)x]&4)!=0) +#define qrfAlnum(x) ((qrfCType[(unsigned char)x]&6)!=0) + +#ifndef deliberate_fall_through +/* Quiet some compilers about some of our intentional code. */ +# if defined(GCC_VERSION) && GCC_VERSION>=7000000 +# define deliberate_fall_through __attribute__((fallthrough)); +# else +# define deliberate_fall_through +# endif +#endif + +/* +** Set an error code and error message. +*/ +static void qrfError( + Qrf *p, /* Query result state */ + int iCode, /* Error code */ + const char *zFormat, /* Message format (or NULL) */ + ... +){ + p->iErr = iCode; + if( p->pzErr!=0 ){ + sqlite3_free(*p->pzErr); + *p->pzErr = 0; + if( zFormat ){ + va_list ap; + va_start(ap, zFormat); + *p->pzErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + } + } } /* -** Write I/O traces to the following stream. +** Out-of-memory error. */ -#ifdef SQLITE_ENABLE_IOTRACE -static FILE *iotrace = 0; -#endif +static void qrfOom(Qrf *p){ + qrfError(p, SQLITE_NOMEM, "out of memory"); +} /* -** This routine works like printf in that its first argument is a -** format string and subsequent arguments are values to be substituted -** in place of % fields. The result of formatting this string -** is written to iotrace. +** Transfer any error in pStr over into p. */ -#ifdef SQLITE_ENABLE_IOTRACE -static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ - va_list ap; - char *z; - if( iotrace==0 ) return; - va_start(ap, zFormat); - z = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - utf8_printf(iotrace, "%s", z); - sqlite3_free(z); +static void qrfStrErr(Qrf *p, sqlite3_str *pStr){ + int rc = pStr ? sqlite3_str_errcode(pStr) : 0; + if( rc ){ + qrfError(p, rc, sqlite3_errstr(rc)); + } } -#endif + /* -** Output string zUtf to stream pOut as w characters. If w is negative, -** then right-justify the text. W is the width in UTF-8 characters, not -** in bytes. This is different from the %*.*s specification in printf -** since with %*.*s the width is measured in bytes, not characters. +** Add a new entry to the EXPLAIN QUERY PLAN data */ -static void utf8_width_print(FILE *pOut, int w, const char *zUtf){ - int i; - int n; - int aw = w<0 ? -w : w; - char zBuf[1000]; - if( aw>(int)sizeof(zBuf)/3 ) aw = (int)sizeof(zBuf)/3; - for(i=n=0; zUtf[i]; i++){ - if( (zUtf[i]&0xc0)!=0x80 ){ - n++; - if( n==aw ){ - do{ i++; }while( (zUtf[i]&0xc0)==0x80 ); - break; - } +static void qrfEqpAppend(Qrf *p, int iEqpId, int p2, const char *zText){ + qrfEQPGraphRow *pNew; + sqlite3_int64 nText; + if( zText==0 ) return; + if( p->u.pGraph==0 ){ + p->u.pGraph = sqlite3_malloc64( sizeof(qrfEQPGraph) ); + if( p->u.pGraph==0 ){ + qrfOom(p); + return; } + memset(p->u.pGraph, 0, sizeof(qrfEQPGraph) ); } - if( n>=aw ){ - utf8_printf(pOut, "%.*s", i, zUtf); - }else if( w<0 ){ - utf8_printf(pOut, "%*s%s", aw-n, "", zUtf); + nText = strlen(zText); + pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); + if( pNew==0 ){ + qrfOom(p); + return; + } + pNew->iEqpId = iEqpId; + pNew->iParentId = p2; + memcpy(pNew->zText, zText, nText+1); + pNew->pNext = 0; + if( p->u.pGraph->pLast ){ + p->u.pGraph->pLast->pNext = pNew; }else{ - utf8_printf(pOut, "%s%*s", zUtf, aw-n, ""); + p->u.pGraph->pRow = pNew; } + p->u.pGraph->pLast = pNew; } - /* -** Determines if a string is a number of not. +** Free and reset the EXPLAIN QUERY PLAN data that has been collected +** in p->u.pGraph. */ -static int isNumber(const char *z, int *realnum){ - if( *z=='-' || *z=='+' ) z++; - if( !IsDigit(*z) ){ - return 0; +static void qrfEqpReset(Qrf *p){ + qrfEQPGraphRow *pRow, *pNext; + if( p->u.pGraph ){ + for(pRow = p->u.pGraph->pRow; pRow; pRow = pNext){ + pNext = pRow->pNext; + sqlite3_free(pRow); + } + sqlite3_free(p->u.pGraph); + p->u.pGraph = 0; } - z++; - if( realnum ) *realnum = 0; - while( IsDigit(*z) ){ z++; } - if( *z=='.' ){ - z++; - if( !IsDigit(*z) ) return 0; - while( IsDigit(*z) ){ z++; } - if( realnum ) *realnum = 1; - } - if( *z=='e' || *z=='E' ){ - z++; - if( *z=='+' || *z=='-' ) z++; - if( !IsDigit(*z) ) return 0; - while( IsDigit(*z) ){ z++; } - if( realnum ) *realnum = 1; +} + +/* Return the next EXPLAIN QUERY PLAN line with iEqpId that occurs after +** pOld, or return the first such line if pOld is NULL +*/ +static qrfEQPGraphRow *qrfEqpNextRow(Qrf *p, int iEqpId, qrfEQPGraphRow *pOld){ + qrfEQPGraphRow *pRow = pOld ? pOld->pNext : p->u.pGraph->pRow; + while( pRow && pRow->iParentId!=iEqpId ) pRow = pRow->pNext; + return pRow; +} + +/* Render a single level of the graph that has iEqpId as its parent. Called +** recursively to render sublevels. +*/ +static void qrfEqpRenderLevel(Qrf *p, int iEqpId){ + qrfEQPGraphRow *pRow, *pNext; + i64 n = strlen(p->u.pGraph->zPrefix); + char *z; + for(pRow = qrfEqpNextRow(p, iEqpId, 0); pRow; pRow = pNext){ + pNext = qrfEqpNextRow(p, iEqpId, pRow); + z = pRow->zText; + sqlite3_str_appendf(p->pOut, "%s%s%s\n", p->u.pGraph->zPrefix, + pNext ? "|--" : "`--", z); + if( n<(i64)sizeof(p->u.pGraph->zPrefix)-7 ){ + memcpy(&p->u.pGraph->zPrefix[n], pNext ? "| " : " ", 4); + qrfEqpRenderLevel(p, pRow->iEqpId); + p->u.pGraph->zPrefix[n] = 0; + } } - return *z==0; } /* -** Compute a string length that is limited to what can be stored in -** lower 30 bits of a 32-bit signed integer. +** Render the 64-bit value N in a more human-readable format into +** pOut. +** +** + Only show the first three significant digits. +** + Append suffixes K, M, G, T, P, and E for 1e3, 1e6, ... 1e18 */ -static int strlen30(const char *z){ - const char *z2 = z; - while( *z2 ){ z2++; } - return 0x3fffffff & (int)(z2 - z); +static void qrfApproxInt64(sqlite3_str *pOut, i64 N){ + static const char aSuffix[] = { 'K', 'M', 'G', 'T', 'P', 'E' }; + int i; + if( N<0 ){ + N = N==INT64_MIN ? INT64_MAX : -N; + sqlite3_str_append(pOut, "-", 1); + } + if( N<10000 ){ + sqlite3_str_appendf(pOut, "%4lld ", N); + return; + } + for(i=1; i<=18; i++){ + N = (N+5)/10; + if( N<10000 ){ + int n = (int)N; + switch( i%3 ){ + case 0: + sqlite3_str_appendf(pOut, "%d.%02d", n/1000, (n%1000)/10); + break; + case 1: + sqlite3_str_appendf(pOut, "%2d.%d", n/100, (n%100)/10); + break; + case 2: + sqlite3_str_appendf(pOut, "%4d", n/10); + break; + } + sqlite3_str_append(pOut, &aSuffix[i/3], 1); + break; + } + } } /* -** Return the length of a string in characters. Multibyte UTF8 characters -** count as a single character. +** Display and reset the EXPLAIN QUERY PLAN data */ -static int strlenChar(const char *z){ - int n = 0; - while( *z ){ - if( (0xc0&*(z++))!=0x80 ) n++; +static void qrfEqpRender(Qrf *p, i64 nCycle){ + qrfEQPGraphRow *pRow; + if( p->u.pGraph!=0 && (pRow = p->u.pGraph->pRow)!=0 ){ + if( pRow->zText[0]=='-' ){ + if( pRow->pNext==0 ){ + qrfEqpReset(p); + return; + } + sqlite3_str_appendf(p->pOut, "%s\n", pRow->zText+3); + p->u.pGraph->pRow = pRow->pNext; + sqlite3_free(pRow); + }else if( nCycle>0 ){ + int nSp = p->u.pGraph->nWidth - 2; + if( p->spec.eStyle==QRF_STYLE_StatsEst ){ + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "Cycles Loops (est) Rows (est)\n"); + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "---------- ------------ ------------\n"); + }else{ + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "Cycles Loops Rows \n"); + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "---------- ----- -----\n"); + } + sqlite3_str_appendall(p->pOut, "QUERY PLAN"); + sqlite3_str_appendchar(p->pOut, nSp - 10, ' '); + qrfApproxInt64(p->pOut, nCycle); + sqlite3_str_appendall(p->pOut, " 100%\n"); + }else{ + sqlite3_str_appendall(p->pOut, "QUERY PLAN\n"); + } + p->u.pGraph->zPrefix[0] = 0; + qrfEqpRenderLevel(p, 0); + qrfEqpReset(p); } - return n; } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS /* -** This routine reads a line of text from FILE in, stores -** the text in memory obtained from malloc() and returns a pointer -** to the text. NULL is returned at end of file, or if malloc() -** fails. +** Helper function for qrfExpStats(). ** -** If zLine is not NULL then it is a malloced buffer returned from -** a previous call to this routine that may be reused. */ -static char *local_getline(char *zLine, FILE *in){ - int nLine = zLine==0 ? 0 : 100; - int n = 0; - - while( 1 ){ - if( n+100>nLine ){ - nLine = nLine*2 + 100; - zLine = realloc(zLine, nLine); - if( zLine==0 ) shell_out_of_memory(); - } - if( fgets(&zLine[n], nLine - n, in)==0 ){ - if( n==0 ){ - free(zLine); - return 0; +static int qrfStatsHeight(sqlite3_stmt *p, int iEntry){ + int iPid = 0; + int ret = 1; + sqlite3_stmt_scanstatus_v2(p, iEntry, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + while( iPid!=0 ){ + int ii; + for(ii=0; 1; ii++){ + int iId; + int res; + res = sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iId + ); + if( res ) break; + if( iId==iPid ){ + sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_PARENTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); } - zLine[n] = 0; - break; } - while( zLine[n] ) n++; - if( n>0 && zLine[n-1]=='\n' ){ - n--; - if( n>0 && zLine[n-1]=='\r' ) n--; - zLine[n] = 0; + ret++; + } + return ret; +} +#endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ + + +/* +** Generate ".scanstatus est" style of EQP output. +*/ +static void qrfEqpStats(Qrf *p){ +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + qrfError(p, SQLITE_ERROR, "not available in this build"); +#else + static const int f = SQLITE_SCANSTAT_COMPLEX; + sqlite3_stmt *pS = p->pStmt; + int i = 0; + i64 nTotal = 0; + int nWidth = 0; + int prevPid = -1; /* Previous iPid */ + double rEstCum = 1.0; /* Cumulative row estimate */ + sqlite3_str *pLine = sqlite3_str_new(p->db); + sqlite3_str *pStats = sqlite3_str_new(p->db); + qrfEqpReset(p); + + for(i=0; 1; i++){ + const char *z = 0; + int n = 0; + if( sqlite3_stmt_scanstatus_v2(pS,i,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ break; } + n = (int)strlen(z) + qrfStatsHeight(pS,i)*3; + if( n>nWidth ) nWidth = n; } -#if defined(_WIN32) || defined(WIN32) - /* For interactive input on Windows systems, translate the - ** multi-byte characterset characters into UTF-8. */ - if( stdin_is_interactive && in==stdin ){ - char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); - if( zTrans ){ - int nTrans = strlen30(zTrans)+1; - if( nTrans>nLine ){ - zLine = realloc(zLine, nTrans); - if( zLine==0 ) shell_out_of_memory(); + nWidth += 2; + + sqlite3_stmt_scanstatus_v2(pS,-1, SQLITE_SCANSTAT_NCYCLE, f, (void*)&nTotal); + for(i=0; 1; i++){ + i64 nLoop = 0; + i64 nRow = 0; + i64 nCycle = 0; + int iId = 0; + int iPid = 0; + const char *zo = 0; + const char *zName = 0; + double rEst = 0.0; + + if( sqlite3_stmt_scanstatus_v2(pS,i,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&zo) ){ + break; + } + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); + if( iPid!=prevPid ){ + prevPid = iPid; + rEstCum = 1.0; + } + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_EST,f,(void*)&rEst); + rEstCum *= rEst; + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NLOOP,f,(void*)&nLoop); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NVISIT,f,(void*)&nRow); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NCYCLE,f,(void*)&nCycle); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_SELECTID,f,(void*)&iId); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NAME,f,(void*)&zName); + + if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ + int nSp = 0; + sqlite3_str_reset(pStats); + if( nCycle>=0 && nTotal>0 ){ + qrfApproxInt64(pStats, nCycle); + sqlite3_str_appendf(pStats, " %3d%%", + ((nCycle*100)+nTotal/2) / nTotal + ); + nSp = 2; + } + if( nLoop>=0 ){ + if( nSp ) sqlite3_str_appendchar(pStats, nSp, ' '); + qrfApproxInt64(pStats, nLoop); + nSp = 2; + if( p->spec.eStyle==QRF_STYLE_StatsEst ){ + sqlite3_str_appendf(pStats, " "); + qrfApproxInt64(pStats, (i64)(rEstCum/rEst)); + } + } + if( nRow>=0 ){ + if( nSp ) sqlite3_str_appendchar(pStats, nSp, ' '); + qrfApproxInt64(pStats, nRow); + nSp = 2; + if( p->spec.eStyle==QRF_STYLE_StatsEst ){ + sqlite3_str_appendf(pStats, " "); + qrfApproxInt64(pStats, (i64)rEstCum); + } } - memcpy(zLine, zTrans, nTrans); - sqlite3_free(zTrans); + sqlite3_str_appendf(pLine, + "% *s %s", -1*(nWidth-qrfStatsHeight(pS,i)*3), zo, + sqlite3_str_value(pStats) + ); + sqlite3_str_reset(pStats); + qrfEqpAppend(p, iId, iPid, sqlite3_str_value(pLine)); + sqlite3_str_reset(pLine); + }else{ + qrfEqpAppend(p, iId, iPid, zo); } } -#endif /* defined(_WIN32) || defined(WIN32) */ - return zLine; + if( p->u.pGraph ) p->u.pGraph->nWidth = nWidth; + qrfStrErr(p, pLine); + sqlite3_free(sqlite3_str_finish(pLine)); + qrfStrErr(p, pStats); + sqlite3_free(sqlite3_str_finish(pStats)); +#endif } + /* -** Retrieve a single line of input text. -** -** If in==0 then read from standard input and prompt before each line. -** If isContinuation is true, then a continuation prompt is appropriate. -** If isContinuation is zero, then the main prompt should be used. -** -** If zPrior is not NULL then it is a buffer from a prior call to this -** routine that can be reused. -** -** The result is stored in space obtained from malloc() and must either -** be freed by the caller or else passed back into this routine via the -** zPrior argument for reuse. +** Reset the prepared statement. */ -static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ - char *zPrompt; - char *zResult; - if( in!=0 ){ - zResult = local_getline(zPrior, in); - }else{ - zPrompt = isContinuation ? continuePrompt : mainPrompt; -#if SHELL_USE_LOCAL_GETLINE - printf("%s", zPrompt); - fflush(stdout); - zResult = local_getline(zPrior, stdin); -#else - free(zPrior); - zResult = shell_readline(zPrompt); - if( zResult && *zResult ) shell_add_history(zResult); -#endif +static void qrfResetStmt(Qrf *p){ + int rc = sqlite3_reset(p->pStmt); + if( rc!=SQLITE_OK && p->iErr==SQLITE_OK ){ + qrfError(p, rc, "%s", sqlite3_errmsg(p->db)); } - return zResult; } - /* -** Return the value of a hexadecimal digit. Return -1 if the input -** is not a hex digit. +** If xWrite is defined, send all content of pOut to xWrite and +** reset pOut. */ -static int hexDigitValue(char c){ - if( c>='0' && c<='9' ) return c - '0'; - if( c>='a' && c<='f' ) return c - 'a' + 10; - if( c>='A' && c<='F' ) return c - 'A' + 10; - return -1; +static void qrfWrite(Qrf *p){ + int n; + if( p->spec.xWrite && (n = sqlite3_str_length(p->pOut))>0 ){ + int rc = p->spec.xWrite(p->spec.pWriteArg, + sqlite3_str_value(p->pOut), + (sqlite3_int64)n); + sqlite3_str_reset(p->pOut); + if( rc ){ + qrfError(p, rc, "Failed to write %d bytes of output", n); + } + } +} + +/* Lookup table to estimate the number of columns consumed by a Unicode +** character. +*/ +static const struct { + unsigned char w; /* Width of the character in columns */ + int iFirst; /* First character in a span having this width */ +} aQrfUWidth[] = { + /* {1, 0x00000}, */ + {0, 0x00300}, {1, 0x00370}, {0, 0x00483}, {1, 0x00487}, {0, 0x00488}, + {1, 0x0048a}, {0, 0x00591}, {1, 0x005be}, {0, 0x005bf}, {1, 0x005c0}, + {0, 0x005c1}, {1, 0x005c3}, {0, 0x005c4}, {1, 0x005c6}, {0, 0x005c7}, + {1, 0x005c8}, {0, 0x00600}, {1, 0x00604}, {0, 0x00610}, {1, 0x00616}, + {0, 0x0064b}, {1, 0x0065f}, {0, 0x00670}, {1, 0x00671}, {0, 0x006d6}, + {1, 0x006e5}, {0, 0x006e7}, {1, 0x006e9}, {0, 0x006ea}, {1, 0x006ee}, + {0, 0x0070f}, {1, 0x00710}, {0, 0x00711}, {1, 0x00712}, {0, 0x00730}, + {1, 0x0074b}, {0, 0x007a6}, {1, 0x007b1}, {0, 0x007eb}, {1, 0x007f4}, + {0, 0x00901}, {1, 0x00903}, {0, 0x0093c}, {1, 0x0093d}, {0, 0x00941}, + {1, 0x00949}, {0, 0x0094d}, {1, 0x0094e}, {0, 0x00951}, {1, 0x00955}, + {0, 0x00962}, {1, 0x00964}, {0, 0x00981}, {1, 0x00982}, {0, 0x009bc}, + {1, 0x009bd}, {0, 0x009c1}, {1, 0x009c5}, {0, 0x009cd}, {1, 0x009ce}, + {0, 0x009e2}, {1, 0x009e4}, {0, 0x00a01}, {1, 0x00a03}, {0, 0x00a3c}, + {1, 0x00a3d}, {0, 0x00a41}, {1, 0x00a43}, {0, 0x00a47}, {1, 0x00a49}, + {0, 0x00a4b}, {1, 0x00a4e}, {0, 0x00a70}, {1, 0x00a72}, {0, 0x00a81}, + {1, 0x00a83}, {0, 0x00abc}, {1, 0x00abd}, {0, 0x00ac1}, {1, 0x00ac6}, + {0, 0x00ac7}, {1, 0x00ac9}, {0, 0x00acd}, {1, 0x00ace}, {0, 0x00ae2}, + {1, 0x00ae4}, {0, 0x00b01}, {1, 0x00b02}, {0, 0x00b3c}, {1, 0x00b3d}, + {0, 0x00b3f}, {1, 0x00b40}, {0, 0x00b41}, {1, 0x00b44}, {0, 0x00b4d}, + {1, 0x00b4e}, {0, 0x00b56}, {1, 0x00b57}, {0, 0x00b82}, {1, 0x00b83}, + {0, 0x00bc0}, {1, 0x00bc1}, {0, 0x00bcd}, {1, 0x00bce}, {0, 0x00c3e}, + {1, 0x00c41}, {0, 0x00c46}, {1, 0x00c49}, {0, 0x00c4a}, {1, 0x00c4e}, + {0, 0x00c55}, {1, 0x00c57}, {0, 0x00cbc}, {1, 0x00cbd}, {0, 0x00cbf}, + {1, 0x00cc0}, {0, 0x00cc6}, {1, 0x00cc7}, {0, 0x00ccc}, {1, 0x00cce}, + {0, 0x00ce2}, {1, 0x00ce4}, {0, 0x00d41}, {1, 0x00d44}, {0, 0x00d4d}, + {1, 0x00d4e}, {0, 0x00dca}, {1, 0x00dcb}, {0, 0x00dd2}, {1, 0x00dd5}, + {0, 0x00dd6}, {1, 0x00dd7}, {0, 0x00e31}, {1, 0x00e32}, {0, 0x00e34}, + {1, 0x00e3b}, {0, 0x00e47}, {1, 0x00e4f}, {0, 0x00eb1}, {1, 0x00eb2}, + {0, 0x00eb4}, {1, 0x00eba}, {0, 0x00ebb}, {1, 0x00ebd}, {0, 0x00ec8}, + {1, 0x00ece}, {0, 0x00f18}, {1, 0x00f1a}, {0, 0x00f35}, {1, 0x00f36}, + {0, 0x00f37}, {1, 0x00f38}, {0, 0x00f39}, {1, 0x00f3a}, {0, 0x00f71}, + {1, 0x00f7f}, {0, 0x00f80}, {1, 0x00f85}, {0, 0x00f86}, {1, 0x00f88}, + {0, 0x00f90}, {1, 0x00f98}, {0, 0x00f99}, {1, 0x00fbd}, {0, 0x00fc6}, + {1, 0x00fc7}, {0, 0x0102d}, {1, 0x01031}, {0, 0x01032}, {1, 0x01033}, + {0, 0x01036}, {1, 0x0103b}, {0, 0x01058}, + {1, 0x0105a}, {2, 0x01100}, {0, 0x01160}, {1, 0x01200}, {0, 0x0135f}, + {1, 0x01360}, {0, 0x01712}, {1, 0x01715}, {0, 0x01732}, {1, 0x01735}, + {0, 0x01752}, {1, 0x01754}, {0, 0x01772}, {1, 0x01774}, {0, 0x017b4}, + {1, 0x017b6}, {0, 0x017b7}, {1, 0x017be}, {0, 0x017c6}, {1, 0x017c7}, + {0, 0x017c9}, {1, 0x017d4}, {0, 0x017dd}, {1, 0x017de}, {0, 0x0180b}, + {1, 0x0180e}, {0, 0x018a9}, {1, 0x018aa}, {0, 0x01920}, {1, 0x01923}, + {0, 0x01927}, {1, 0x01929}, {0, 0x01932}, {1, 0x01933}, {0, 0x01939}, + {1, 0x0193c}, {0, 0x01a17}, {1, 0x01a19}, {0, 0x01b00}, {1, 0x01b04}, + {0, 0x01b34}, {1, 0x01b35}, {0, 0x01b36}, {1, 0x01b3b}, {0, 0x01b3c}, + {1, 0x01b3d}, {0, 0x01b42}, {1, 0x01b43}, {0, 0x01b6b}, {1, 0x01b74}, + {0, 0x01dc0}, {1, 0x01dcb}, {0, 0x01dfe}, {1, 0x01e00}, {0, 0x0200b}, + {1, 0x02010}, {0, 0x0202a}, {1, 0x0202f}, {0, 0x02060}, {1, 0x02064}, + {0, 0x0206a}, {1, 0x02070}, {0, 0x020d0}, {1, 0x020f0}, {2, 0x02329}, + {1, 0x0232b}, {2, 0x02e80}, {0, 0x0302a}, {2, 0x03030}, {1, 0x0303f}, + {2, 0x03040}, {0, 0x03099}, {2, 0x0309b}, {1, 0x0a4d0}, {0, 0x0a806}, + {1, 0x0a807}, {0, 0x0a80b}, {1, 0x0a80c}, {0, 0x0a825}, {1, 0x0a827}, + {2, 0x0ac00}, {1, 0x0d7a4}, {2, 0x0f900}, {1, 0x0fb00}, {0, 0x0fb1e}, + {1, 0x0fb1f}, {0, 0x0fe00}, {2, 0x0fe10}, {1, 0x0fe1a}, {0, 0x0fe20}, + {1, 0x0fe24}, {2, 0x0fe30}, {1, 0x0fe70}, {0, 0x0feff}, {2, 0x0ff00}, + {1, 0x0ff61}, {2, 0x0ffe0}, {1, 0x0ffe7}, {0, 0x0fff9}, {1, 0x0fffc}, + {0, 0x10a01}, {1, 0x10a04}, {0, 0x10a05}, {1, 0x10a07}, {0, 0x10a0c}, + {1, 0x10a10}, {0, 0x10a38}, {1, 0x10a3b}, {0, 0x10a3f}, {1, 0x10a40}, + {0, 0x1d167}, {1, 0x1d16a}, {0, 0x1d173}, {1, 0x1d183}, {0, 0x1d185}, + {1, 0x1d18c}, {0, 0x1d1aa}, {1, 0x1d1ae}, {0, 0x1d242}, {1, 0x1d245}, + {2, 0x20000}, {1, 0x2fffe}, {2, 0x30000}, {1, 0x3fffe}, {0, 0xe0001}, + {1, 0xe0002}, {0, 0xe0020}, {1, 0xe0080}, {0, 0xe0100}, {1, 0xe01f0} +}; + +/* +** Return an estimate of the width, in columns, for the single Unicode +** character c. For normal characters, the answer is always 1. But the +** estimate might be 0 or 2 for zero-width and double-width characters. +** +** Different display devices display unicode using different widths. So +** it is impossible to know that true display width with 100% accuracy. +** Inaccuracies in the width estimates might cause columns to be misaligned. +** Unfortunately, there is nothing we can do about that. +*/ +int sqlite3_qrf_wcwidth(int c){ + int iFirst, iLast; + + /* Fast path for common characters */ + if( c<0x300 ) return 1; + + /* The general case */ + iFirst = 0; + iLast = sizeof(aQrfUWidth)/sizeof(aQrfUWidth[0]) - 1; + while( iFirst c ){ + iLast = iMid - 1; + }else{ + return aQrfUWidth[iMid].w; + } + } + if( aQrfUWidth[iLast].iFirst > c ) return aQrfUWidth[iFirst].w; + return aQrfUWidth[iLast].w; } /* -** Interpret zArg as an integer value, possibly with suffixes. +** Compute the value and length of a multi-byte UTF-8 character that +** begins at z[0]. Return the length. Write the Unicode value into *pU. +** +** This routine only works for *multi-byte* UTF-8 characters. It does +** not attempt to detect illegal characters. */ -static sqlite3_int64 integerValue(const char *zArg){ - sqlite3_int64 v = 0; - static const struct { char *zSuffix; int iMult; } aMult[] = { - { "KiB", 1024 }, - { "MiB", 1024*1024 }, - { "GiB", 1024*1024*1024 }, - { "KB", 1000 }, - { "MB", 1000000 }, - { "GB", 1000000000 }, - { "K", 1000 }, - { "M", 1000000 }, - { "G", 1000000000 }, - }; - int i; - int isNeg = 0; - if( zArg[0]=='-' ){ - isNeg = 1; - zArg++; - }else if( zArg[0]=='+' ){ - zArg++; +int sqlite3_qrf_decode_utf8(const unsigned char *z, int *pU){ + if( (z[0] & 0xe0)==0xc0 && (z[1] & 0xc0)==0x80 ){ + *pU = ((z[0] & 0x1f)<<6) | (z[1] & 0x3f); + return 2; } - if( zArg[0]=='0' && zArg[1]=='x' ){ - int x; - zArg += 2; - while( (x = hexDigitValue(zArg[0]))>=0 ){ - v = (v<<4) + x; - zArg++; - } - }else{ - while( IsDigit(zArg[0]) ){ - v = v*10 + zArg[0] - '0'; - zArg++; - } + if( (z[0] & 0xf0)==0xe0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 ){ + *pU = ((z[0] & 0x0f)<<12) | ((z[1] & 0x3f)<<6) | (z[2] & 0x3f); + return 3; } - for(i=0; i=0x30 && z[i]<=0x3f ){ i++; } + while( z[i]>=0x20 && z[i]<=0x2f ){ i++; } + if( z[i]<0x40 || z[i]>0x7e ) return 0; + return i+1; +} /* -** Initialize and destroy a ShellText object -*/ -static void initText(ShellText *p){ - memset(p, 0, sizeof(*p)); +** Return the length of a string in display characters. +** +** Most characters of the input string count as 1, including +** multi-byte UTF8 characters. However, zero-width unicode +** characters and VT100 escape sequences count as zero, and +** double-width characters count as two. +** +** The definition of "zero-width" and "double-width" characters +** is not precise. It depends on the output device, to some extent, +** and it varies according to the Unicode version. This routine +** makes the best guess that it can. +*/ +size_t sqlite3_qrf_wcswidth(const char *zIn){ + const unsigned char *z = (const unsigned char*)zIn; + size_t n = 0; + while( *z ){ + if( z[0]<' ' ){ + int k; + if( z[0]=='\033' && (k = qrfIsVt100(z))>0 ){ + z += k; + }else{ + z++; + } + }else if( (0x80&z[0])==0 ){ + n++; + z++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(z, &u); + z += len; + n += sqlite3_qrf_wcwidth(u); + } + } + return n; } -static void freeText(ShellText *p){ - free(p->z); - initText(p); + +/* +** Return the display width of the longest line of text +** in the (possibly) multi-line input string zIn[0..nByte]. +** zIn[] is not necessarily zero-terminated. Take +** into account tab characters, zero- and double-width +** characters, CR and NL, and VT100 escape codes. +** +** Write the number of newlines into *pnNL. So, *pnNL will +** return 0 if everything fits on one line, or positive it +** it will need to be split. +*/ +static int qrfDisplayWidth(const char *zIn, sqlite3_int64 nByte, int *pnNL){ + const unsigned char *z; + const unsigned char *zEnd; + int mx = 0; + int n = 0; + int nNL = 0; + if( zIn==0 ) zIn = ""; + z = (const unsigned char*)zIn; + zEnd = &z[nByte]; + while( z0 ){ + z += k; + }else{ + if( z[0]=='\t' ){ + n = (n+8)&~7; + }else if( z[0]=='\n' || z[0]=='\r' ){ + nNL++; + if( n>mx ) mx = n; + n = 0; + } + z++; + } + }else if( (0x80&z[0])==0 ){ + n++; + z++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(z, &u); + z += len; + n += sqlite3_qrf_wcwidth(u); + } + } + if( mx>n ) n = mx; + if( pnNL ) *pnNL = nNL; + return n; } -/* zIn is either a pointer to a NULL-terminated string in memory obtained -** from malloc(), or a NULL pointer. The string pointed to by zAppend is -** added to zIn, and the result returned in memory obtained from malloc(). -** zIn, if it was not NULL, is freed. +/* +** Escape the input string if it is needed and in accordance with +** eEsc, which is either QRF_ESC_Ascii or QRF_ESC_Symbol. ** -** If the third argument, quote, is not '\0', then it is used as a -** quote character for zAppend. +** Escaping is needed if the string contains any control characters +** other than \t, \n, and \r\n +** +** If no escaping is needed (the common case) then set *ppOut to NULL +** and return 0. If escaping is needed, write the escaped string into +** memory obtained from sqlite3_malloc64() and make *ppOut point to that +** memory and return 0. If an error occurs, return non-zero. +** +** The caller is responsible for freeing *ppFree if it is non-NULL in order +** to reclaim memory. */ -static void appendText(ShellText *p, char const *zAppend, char quote){ - int len; - int i; - int nAppend = strlen30(zAppend); - - len = nAppend+p->n+1; - if( quote ){ - len += 2; - for(i=0; i0x1f + || c=='\t' + || c=='\n' + || (c=='\r' && zIn[i+1]=='\n') + ){ + continue; + } + if( i>0 ){ + memmove(&zOut[j], zIn, i); + j += i; + } + zIn += i+1; + i = -1; + if( eEsc==QRF_ESC_Symbol ){ + zOut[j++] = 0xe2; + zOut[j++] = 0x90; + zOut[j++] = 0x80+c; + }else{ + zOut[j++] = '^'; + zOut[j++] = 0x40+c; } } +} - if( p->n+len>=p->nAlloc ){ - p->nAlloc = p->nAlloc*2 + len + 20; - p->z = realloc(p->z, p->nAlloc); - if( p->z==0 ) shell_out_of_memory(); - } +/* +** Determine if the string z[] can be shown as plain text. Return true +** if z[] is unambiguously text. Return false if z[] needs to be +** quoted. +** +** All of the following must be true in order for z[] to be relaxable: +** +** (1) z[] does not begin or end with ' or whitespace +** (2) z[] is not the same as the NULL rendering +** (3) z[] does not looks like a numeric literal +*/ +static int qrfRelaxable(Qrf *p, const char *z){ + size_t i, n; + if( z[0]=='\'' || qrfSpace(z[0]) ) return 0; + if( z[0]==0 ){ + return (p->spec.zNull!=0 && p->spec.zNull[0]!=0); + } + n = strlen(z); + if( n==0 || z[n-1]=='\'' || qrfSpace(z[n-1]) ) return 0; + if( p->spec.zNull && strcmp(p->spec.zNull,z)==0 ) return 0; + i = (z[0]=='-' || z[0]=='+'); + if( strcmp(z+i,"Inf")==0 ) return 0; + if( !qrfDigit(z[i]) ) return 1; + i++; + while( qrfDigit(z[i]) ){ i++; } + if( z[i]==0 ) return 0; + if( z[i]=='.' ){ + i++; + while( qrfDigit(z[i]) ){ i++; } + if( z[i]==0 ) return 0; + } + if( z[i]=='e' || z[i]=='E' ){ + i++; + if( z[i]=='+' || z[i]=='-' ){ i++; } + if( !qrfDigit(z[i]) ) return 1; + i++; + while( qrfDigit(z[i]) ){ i++; } + } + return z[i]!=0; +} - if( quote ){ - char *zCsr = p->z+p->n; - *zCsr++ = quote; - for(i=0; ispec.eText ){ + case QRF_TEXT_Relaxed: + if( qrfRelaxable(p, zTxt) ){ + sqlite3_str_appendall(pOut, zTxt); + break; + } + deliberate_fall_through; /* FALLTHRU */ + case QRF_TEXT_Sql: { + if( p->spec.eEsc==QRF_ESC_Off ){ + sqlite3_str_appendf(pOut, "%Q", zTxt); + }else{ + sqlite3_str_appendf(pOut, "%#Q", zTxt); + } + break; + } + case QRF_TEXT_Csv: { + unsigned int i; + for(i=0; zTxt[i]; i++){ + if( qrfCsvQuote[((const unsigned char*)zTxt)[i]] ){ + i = 0; + break; + } + } + if( i==0 || strstr(zTxt, p->spec.zColumnSep)!=0 ){ + sqlite3_str_appendf(pOut, "\"%w\"", zTxt); + }else{ + sqlite3_str_appendall(pOut, zTxt); + } + break; + } + case QRF_TEXT_Html: { + const unsigned char *z = (const unsigned char*)zTxt; + while( *z ){ + unsigned int i = 0; + unsigned char c; + while( (c=z[i])>'>' + || (c && c!='<' && c!='>' && c!='&' && c!='\"' && c!='\'') + ){ + i++; + } + if( i>0 ){ + sqlite3_str_append(pOut, (const char*)z, i); + } + switch( z[i] ){ + case '>': sqlite3_str_append(pOut, "<", 4); break; + case '&': sqlite3_str_append(pOut, "&", 5); break; + case '<': sqlite3_str_append(pOut, "<", 4); break; + case '"': sqlite3_str_append(pOut, """, 6); break; + case '\'': sqlite3_str_append(pOut, "'", 5); break; + default: i--; + } + z += i + 1; + } + break; + } + case QRF_TEXT_Tcl: + case QRF_TEXT_Json: { + const unsigned char *z = (const unsigned char*)zTxt; + sqlite3_str_append(pOut, "\"", 1); + while( *z ){ + unsigned int i; + for(i=0; z[i]>=0x20 && z[i]!='\\' && z[i]!='"'; i++){} + if( i>0 ){ + sqlite3_str_append(pOut, (const char*)z, i); + } + if( z[i]==0 ) break; + switch( z[i] ){ + case '"': sqlite3_str_append(pOut, "\\\"", 2); break; + case '\\': sqlite3_str_append(pOut, "\\\\", 2); break; + case '\b': sqlite3_str_append(pOut, "\\b", 2); break; + case '\f': sqlite3_str_append(pOut, "\\f", 2); break; + case '\n': sqlite3_str_append(pOut, "\\n", 2); break; + case '\r': sqlite3_str_append(pOut, "\\r", 2); break; + case '\t': sqlite3_str_append(pOut, "\\t", 2); break; + default: { + if( p->spec.eText==QRF_TEXT_Json ){ + sqlite3_str_appendf(pOut, "\\u%04x", z[i]); + }else{ + sqlite3_str_appendf(pOut, "\\%03o", z[i]); + } + break; + } + } + z += i + 1; + } + sqlite3_str_append(pOut, "\"", 1); + break; + } + default: { + sqlite3_str_appendall(pOut, zTxt); + break; } - *zCsr++ = quote; - p->n = (int)(zCsr - p->z); - *zCsr = '\0'; - }else{ - memcpy(p->z+p->n, zAppend, nAppend); - p->n += nAppend; - p->z[p->n] = '\0'; + } + if( p->spec.eEsc!=QRF_ESC_Off ){ + qrfEscape(p->spec.eEsc, pOut, iStart); } } /* -** Attempt to determine if identifier zName needs to be quoted, either -** because it contains non-alphanumeric characters, or because it is an -** SQLite keyword. Be conservative in this estimate: When in doubt assume -** that quoting is required. +** Do a quick sanity check to see aBlob[0..nBlob-1] is valid JSONB +** return true if it is and false if it is not. ** -** Return '"' if quoting is required. Return 0 if no quoting is required. +** False positives are possible, but not false negatives. */ -static char quoteChar(const char *zName){ - int i; - if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; - for(i=0; zName[i]; i++){ - if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; - } - return sqlite3_keyword_check(zName, i) ? '"' : 0; +static int qrfJsonbQuickCheck(unsigned char *aBlob, int nBlob){ + unsigned char x; /* Payload size half-byte */ + int i; /* Loop counter */ + int n; /* Bytes in the payload size integer */ + sqlite3_uint64 sz; /* value of the payload size integer */ + + if( nBlob==0 ) return 0; + x = aBlob[0]>>4; + if( x<=11 ) return nBlob==(1+x); + n = x<14 ? x-11 : 4*(x-13); + if( nBlob<1+n ) return 0; + sz = aBlob[1]; + for(i=1; ipStmt is known to be a BLOB. Check +** to see if that BLOB is really a JSONB blob. If it is, then translate +** it into a text JSON representation and return a pointer to that text JSON. +** If the BLOB is not JSONB, then return a NULL pointer. +** +** The memory used to hold the JSON text is managed internally by the +** "p" object and is overwritten and/or deallocated upon the next call +** to this routine (with the same p argument) or when the p object is +** finailized. */ -static char *shellFakeSchema( - sqlite3 *db, /* The database connection containing the vtab */ - const char *zSchema, /* Schema of the database holding the vtab */ - const char *zName /* The name of the virtual table */ -){ - sqlite3_stmt *pStmt = 0; - char *zSql; - ShellText s; - char cQuote; - char *zDiv = "("; - int nRow = 0; - - zSql = sqlite3_mprintf("PRAGMA \"%w\".table_info=%Q;", - zSchema ? zSchema : "main", zName); - sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - initText(&s); - if( zSchema ){ - cQuote = quoteChar(zSchema); - if( cQuote && sqlite3_stricmp(zSchema,"temp")==0 ) cQuote = 0; - appendText(&s, zSchema, cQuote); - appendText(&s, ".", 0); +static const char *qrfJsonbToJson(Qrf *p, int iCol){ + int nByte; + const void *pBlob; + int rc; + nByte = sqlite3_column_bytes(p->pStmt, iCol); + pBlob = sqlite3_column_blob(p->pStmt, iCol); + if( qrfJsonbQuickCheck((unsigned char*)pBlob, nByte)==0 ){ + return 0; } - cQuote = quoteChar(zName); - appendText(&s, zName, cQuote); - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - const char *zCol = (const char*)sqlite3_column_text(pStmt, 1); - nRow++; - appendText(&s, zDiv, 0); - zDiv = ","; - cQuote = quoteChar(zCol); - appendText(&s, zCol, cQuote); + if( p->pJTrans==0 ){ + sqlite3 *db; + rc = sqlite3_open(":memory:",&db); + if( rc ){ + sqlite3_close(db); + return 0; + } + rc = sqlite3_prepare_v2(db, "SELECT json(?1)", -1, &p->pJTrans, 0); + if( rc ){ + sqlite3_finalize(p->pJTrans); + p->pJTrans = 0; + sqlite3_close(db); + return 0; + } + }else{ + sqlite3_reset(p->pJTrans); } - appendText(&s, ")", 0); - sqlite3_finalize(pStmt); - if( nRow==0 ){ - freeText(&s); - s.z = 0; + sqlite3_bind_blob(p->pJTrans, 1, (void*)pBlob, nByte, SQLITE_STATIC); + rc = sqlite3_step(p->pJTrans); + if( rc==SQLITE_ROW ){ + return (const char*)sqlite3_column_text(p->pJTrans, 0); + }else{ + return 0; } - return s.z; } /* -** SQL function: shell_module_schema(X) +** Adjust the input string zIn[] such that it is no more than N display +** characters wide. If it is wider than that, then truncate and add +** ellipsis. Or if zIn[] contains a \r or \n, truncate at that point, +** adding ellipsis. Embedded tabs in zIn[] are converted into ordinary +** spaces. ** -** Return a fake schema for the table-valued function or eponymous virtual -** table X. +** Return this display width of the modified title string. */ -static void shellModuleSchema( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal -){ - const char *zName = (const char*)sqlite3_value_text(apVal[0]); - char *zFake = shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName); - UNUSED_PARAMETER(nVal); - if( zFake ){ - sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake), - -1, sqlite3_free); - free(zFake); +static int qrfTitleLimit(char *zIn, int N){ + unsigned char *z = (unsigned char*)zIn; + int n = 0; + unsigned char *zEllipsis = 0; + while( 1 /*exit-by-break*/ ){ + if( z[0]<' ' ){ + int k; + if( z[0]==0 ){ + zEllipsis = 0; + break; + }else if( z[0]=='\033' && (k = qrfIsVt100(z))>0 ){ + z += k; + }else if( z[0]=='\t' ){ + z[0] = ' '; + }else if( z[0]=='\n' || z[0]=='\r' ){ + z[0] = ' '; + }else{ + z++; + } + }else if( (0x80&z[0])==0 ){ + if( n>=(N-3) && zEllipsis==0 ) zEllipsis = z; + if( n==N ){ z[0] = 0; break; } + n++; + z++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(z, &u); + if( n+len>(N-3) && zEllipsis==0 ) zEllipsis = z; + if( n+len>N ){ z[0] = 0; break; } + z += len; + n += sqlite3_qrf_wcwidth(u); + } } + if( zEllipsis && N>=3 ) memcpy(zEllipsis,"...",4); + return n; } + /* -** SQL function: shell_add_schema(S,X) -** -** Add the schema name X to the CREATE statement in S and return the result. -** Examples: -** -** CREATE TABLE t1(x) -> CREATE TABLE xyz.t1(x); -** -** Also works on -** -** CREATE INDEX -** CREATE UNIQUE INDEX -** CREATE VIEW -** CREATE TRIGGER -** CREATE VIRTUAL TABLE -** -** This UDF is used by the .schema command to insert the schema name of -** attached databases into the middle of the sqlite_master.sql field. +** Render value pVal into pOut */ -static void shellAddSchemaName( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal -){ - static const char *aPrefix[] = { - "TABLE", - "INDEX", - "UNIQUE INDEX", - "VIEW", - "TRIGGER", - "VIRTUAL TABLE" - }; - int i = 0; - const char *zIn = (const char*)sqlite3_value_text(apVal[0]); - const char *zSchema = (const char*)sqlite3_value_text(apVal[1]); - const char *zName = (const char*)sqlite3_value_text(apVal[2]); - sqlite3 *db = sqlite3_context_db_handle(pCtx); - UNUSED_PARAMETER(nVal); - if( zIn!=0 && strncmp(zIn, "CREATE ", 7)==0 ){ - for(i=0; i<(int)(sizeof(aPrefix)/sizeof(aPrefix[0])); i++){ - int n = strlen30(aPrefix[i]); - if( strncmp(zIn+7, aPrefix[i], n)==0 && zIn[n+7]==' ' ){ - char *z = 0; - char *zFake = 0; - if( zSchema ){ - char cQuote = quoteChar(zSchema); - if( cQuote && sqlite3_stricmp(zSchema,"temp")!=0 ){ - z = sqlite3_mprintf("%.*s \"%w\".%s", n+7, zIn, zSchema, zIn+n+8); +static void qrfRenderValue(Qrf *p, sqlite3_str *pOut, int iCol){ +#if SQLITE_VERSION_NUMBER>=3052000 + int iStartLen = sqlite3_str_length(pOut); +#endif + if( p->spec.xRender ){ + sqlite3_value *pVal; + char *z; + pVal = sqlite3_value_dup(sqlite3_column_value(p->pStmt,iCol)); + z = p->spec.xRender(p->spec.pRenderArg, pVal); + sqlite3_value_free(pVal); + if( z ){ + sqlite3_str_appendall(pOut, z); + sqlite3_free(z); + return; + } + } + switch( sqlite3_column_type(p->pStmt,iCol) ){ + case SQLITE_INTEGER: { + sqlite3_str_appendf(pOut, "%lld", sqlite3_column_int64(p->pStmt,iCol)); + break; + } + case SQLITE_FLOAT: { + const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); + sqlite3_str_appendall(pOut, zTxt); + break; + } + case SQLITE_BLOB: { + if( p->spec.bTextJsonb==QRF_Yes ){ + const char *zJson = qrfJsonbToJson(p, iCol); + if( zJson ){ + if( p->spec.eText==QRF_TEXT_Sql ){ + sqlite3_str_append(pOut,"jsonb(",6); + qrfEncodeText(p, pOut, zJson); + sqlite3_str_append(pOut,")",1); }else{ - z = sqlite3_mprintf("%.*s %s.%s", n+7, zIn, zSchema, zIn+n+8); + qrfEncodeText(p, pOut, zJson); } + break; } - if( zName - && aPrefix[i][0]=='V' - && (zFake = shellFakeSchema(db, zSchema, zName))!=0 - ){ - if( z==0 ){ - z = sqlite3_mprintf("%s\n/* %s */", zIn, zFake); - }else{ - z = sqlite3_mprintf("%z\n/* %s */", z, zFake); + } + switch( p->spec.eBlob ){ + case QRF_BLOB_Hex: + case QRF_BLOB_Sql: { + int iStart; + int nBlob = sqlite3_column_bytes(p->pStmt,iCol); + int i, j; + char *zVal; + const unsigned char *a = sqlite3_column_blob(p->pStmt,iCol); + if( p->spec.eBlob==QRF_BLOB_Sql ){ + sqlite3_str_append(pOut, "x'", 2); } - free(zFake); + iStart = sqlite3_str_length(pOut); + sqlite3_str_appendchar(pOut, nBlob, ' '); + sqlite3_str_appendchar(pOut, nBlob, ' '); + if( p->spec.eBlob==QRF_BLOB_Sql ){ + sqlite3_str_appendchar(pOut, 1, '\''); + } + if( sqlite3_str_errcode(pOut) ) return; + zVal = sqlite3_str_value(pOut); + for(i=0, j=iStart; i>4)&0xf]; + zVal[j+1] = "0123456789abcdef"[(c)&0xf]; + } + break; } - if( z ){ - sqlite3_result_text(pCtx, z, -1, sqlite3_free); - return; + case QRF_BLOB_Tcl: + case QRF_BLOB_Json: { + int iStart; + int nBlob = sqlite3_column_bytes(p->pStmt,iCol); + int i, j; + char *zVal; + const unsigned char *a = sqlite3_column_blob(p->pStmt,iCol); + int szC = p->spec.eBlob==QRF_BLOB_Json ? 6 : 4; + sqlite3_str_append(pOut, "\"", 1); + iStart = sqlite3_str_length(pOut); + for(i=szC; i>0; i--){ + sqlite3_str_appendchar(pOut, nBlob, ' '); + } + sqlite3_str_appendchar(pOut, 1, '"'); + if( sqlite3_str_errcode(pOut) ) return; + zVal = sqlite3_str_value(pOut); + for(i=0, j=iStart; i>6)&3); + zVal[j+2] = '0' + ((c>>3)&7); + zVal[j+3] = '0' + (c&7); + }else{ + zVal[j+1] = 'u'; + zVal[j+2] = '0'; + zVal[j+3] = '0'; + zVal[j+4] = "0123456789abcdef"[(c>>4)&0xf]; + zVal[j+5] = "0123456789abcdef"[(c)&0xf]; + } + } + break; + } + case QRF_BLOB_Size: { + int nBlob = sqlite3_column_bytes(p->pStmt,iCol); + sqlite3_str_appendf(pOut, "(%d-byte blob)", nBlob); + break; + } + default: { + const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); + qrfEncodeText(p, pOut, zTxt); + } + } + break; + } + case SQLITE_NULL: { + sqlite3_str_appendall(pOut, p->spec.zNull); + break; + } + case SQLITE_TEXT: { + const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); + qrfEncodeText(p, pOut, zTxt); + break; + } + } +#if SQLITE_VERSION_NUMBER>=3052000 + if( p->spec.nCharLimit>0 + && (sqlite3_str_length(pOut) - iStartLen) > p->spec.nCharLimit + ){ + const unsigned char *z; + int ii = 0, w = 0, limit = p->spec.nCharLimit; + z = (const unsigned char*)sqlite3_str_value(pOut) + iStartLen; + if( limit<4 ) limit = 4; + while( 1 ){ + if( z[ii]<' ' ){ + int k; + if( z[ii]=='\033' && (k = qrfIsVt100(z+ii))>0 ){ + ii += k; + }else if( z[ii]==0 ){ + break; + }else{ + ii++; } + }else if( (0x80&z[ii])==0 ){ + w++; + if( w>limit ) break; + ii++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(&z[ii], &u); + w += sqlite3_qrf_wcwidth(u); + if( w>limit ) break; + ii += len; } } + if( w>limit ){ + sqlite3_str_truncate(pOut, iStartLen+ii); + sqlite3_str_append(pOut, "...", 3); + } } - sqlite3_result_value(pCtx, apVal[0]); +#endif } -/* -** The source code for several run-time loadable extensions is inserted -** below by the ../tool/mkshellc.tcl script. Before processing that included -** code, we need to override some macros to make the included program code -** work here in the middle of this regular program. -*/ -#define SQLITE_EXTENSION_INIT1 -#define SQLITE_EXTENSION_INIT2(X) (void)(X) - -#if defined(_WIN32) && defined(_MSC_VER) -/************************* Begin test_windirent.h ******************/ -/* -** 2015 November 30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains declarations for most of the opendir() family of -** POSIX functions on Win32 using the MSVCRT. -*/ - -#if defined(_WIN32) && defined(_MSC_VER) && !defined(SQLITE_WINDIRENT_H) -#define SQLITE_WINDIRENT_H - -/* -** We need several data types from the Windows SDK header. +/* Trim spaces of the end if pOut */ - -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN +static void qrfRTrim(sqlite3_str *pOut){ +#if SQLITE_VERSION_NUMBER>=3052000 + int nByte = sqlite3_str_length(pOut); + const char *zOut = sqlite3_str_value(pOut); + while( nByte>0 && zOut[nByte-1]==' ' ){ nByte--; } + sqlite3_str_truncate(pOut, nByte); #endif - -#include "windows.h" +} /* -** We need several support functions from the SQLite core. +** Store string zUtf to pOut as w characters. If w is negative, +** then right-justify the text. W is the width in display characters, not +** in bytes. Double-width unicode characters count as two characters. +** VT100 escape sequences count as zero. And so forth. */ - -/* #include "sqlite3.h" */ +static void qrfWidthPrint(Qrf *p, sqlite3_str *pOut, int w, const char *zUtf){ + const unsigned char *a = (const unsigned char*)zUtf; + static const int mxW = 10000000; + unsigned char c; + int i = 0; + int n = 0; + int k; + int aw; + (void)p; + if( w<-mxW ){ + w = -mxW; + }else if( w>mxW ){ + w= mxW; + } + aw = w<0 ? -w : w; + if( a==0 ) a = (const unsigned char*)""; + while( (c = a[i])!=0 ){ + if( (c&0xc0)==0xc0 ){ + int u; + int len = sqlite3_qrf_decode_utf8(a+i, &u); + int x = sqlite3_qrf_wcwidth(u); + if( x+n>aw ){ + break; + } + i += len; + n += x; + }else if( c==0x1b && (k = qrfIsVt100(&a[i]))>0 ){ + i += k; + }else if( n>=aw ){ + break; + }else{ + n++; + i++; + } + } + if( n>=aw ){ + sqlite3_str_append(pOut, zUtf, i); + }else if( w<0 ){ + if( aw>n ) sqlite3_str_appendchar(pOut, aw-n, ' '); + sqlite3_str_append(pOut, zUtf, i); + }else{ + sqlite3_str_append(pOut, zUtf, i); + if( aw>n ) sqlite3_str_appendchar(pOut, aw-n, ' '); + } +} /* -** We need several things from the ANSI and MSVCRT headers. -*/ +** (*pz)[] is a line of text that is to be displayed the box or table or +** similar tabular formats. z[] contain newlines or might be too wide +** to fit in the columns so will need to be split into multiple line. +** +** This routine determines: +** +** * How many bytes of z[] should be shown on the current line. +** * How many character positions those bytes will cover. +** * The byte offset to the start of the next line. +*/ +static void qrfWrapLine( + const char *zIn, /* Input text to be displayed */ + int w, /* Column width in characters (not bytes) */ + int bWrap, /* True if we should do word-wrapping */ + int *pnThis, /* OUT: How many bytes of z[] for the current line */ + int *pnWide, /* OUT: How wide is the text of this line */ + int *piNext /* OUT: Offset into z[] to start of the next line */ +){ + int i; /* Input bytes consumed */ + int k; /* Bytes in a VT100 code */ + int n; /* Output column number */ + const unsigned char *z = (const unsigned char*)zIn; + unsigned char c = 0; + + if( z[0]==0 ){ + *pnThis = 0; + *pnWide = 0; + *piNext = 0; + return; + } + n = 0; + for(i=0; n<=w; i++){ + c = z[i]; + if( c>=0xc0 ){ + int u; + int len = sqlite3_qrf_decode_utf8(&z[i], &u); + int wcw = sqlite3_qrf_wcwidth(u); + if( wcw+n>w ) break; + i += len-1; + n += wcw; + continue; + } + if( c>=' ' ){ + if( n==w ) break; + n++; + continue; + } + if( c==0 || c=='\n' ) break; + if( c=='\r' && z[i+1]=='\n' ){ c = z[++i]; break; } + if( c=='\t' ){ + int wcw = 8 - (n&7); + if( n+wcw>w ) break; + n += wcw; + continue; + } + if( c==0x1b && (k = qrfIsVt100(&z[i]))>0 ){ + i += k-1; + }else if( n==w ){ + break; + }else{ + n++; + } + } + if( c==0 ){ + *pnThis = i; + *pnWide = n; + *piNext = i; + return; + } + if( c=='\n' ){ + *pnThis = i; + *pnWide = n; + *piNext = i+1; + return; + } -#include -#include -#include -#include -#include -#include -#include + /* If we get this far, that means the current line will end at some + ** point that is neither a "\n" or a 0x00. Figure out where that + ** split should occur + */ + if( bWrap && z[i]!=0 && !qrfSpace(z[i]) && qrfAlnum(c)==qrfAlnum(z[i]) ){ + /* Perhaps try to back up to a better place to break the line */ + for(k=i-1; k>=i/2; k--){ + if( qrfSpace(z[k]) ) break; + } + if( k=i/2; k--){ + if( qrfAlnum(z[k-1])!=qrfAlnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; + } + } + if( k>=i/2 ){ + i = k; + n = qrfDisplayWidth((const char*)z, k, 0); + } + } + *pnThis = i; + *pnWide = n; + while( zIn[i]==' ' || zIn[i]=='\t' || zIn[i]=='\r' ){ i++; } + *piNext = i; +} /* -** We may need several defines that should have been in "sys/stat.h". +** Append nVal bytes of text from zVal onto the end of pOut. +** Convert tab characters in zVal to the appropriate number of +** spaces. */ - -#ifndef S_ISREG -#define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) -#endif - -#ifndef S_ISDIR -#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) -#endif - -#ifndef S_ISLNK -#define S_ISLNK(mode) (0) -#endif +static void qrfAppendWithTabs( + sqlite3_str *pOut, /* Append text here */ + const char *zVal, /* Text to append */ + int nVal /* Use only the first nVal bytes of zVal[] */ +){ + int i = 0; + unsigned int col = 0; + unsigned char *z = (unsigned char *)zVal; + while( i0 ){ + sqlite3_str_append(pOut, (const char*)z, k); + z += k; + nVal -= k; + }else if( c=='\t' ){ + k = 8 - (col&7); + sqlite3_str_appendchar(pOut, k, ' '); + col += k; + z++; + nVal--; + }else if( c=='\r' && nVal==1 ){ + z++; + nVal--; + }else{ + char zCtrlPik[4]; + col++; + zCtrlPik[0] = 0xe2; + zCtrlPik[1] = 0x90; + zCtrlPik[2] = 0x80+c; + sqlite3_str_append(pOut, zCtrlPik, 3); + z++; + nVal--; + } + }else if( (0x80&c)==0 ){ + i++; + col++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(&z[i], &u); + i += len; + col += sqlite3_qrf_wcwidth(u); + } + } + sqlite3_str_append(pOut, (const char*)z, i); +} /* -** We may need to provide the "mode_t" type. +** GCC does not define the offsetof() macro so we'll have to do it +** ourselves. */ - -#ifndef MODE_T_DEFINED - #define MODE_T_DEFINED - typedef unsigned short mode_t; +#ifndef offsetof +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) #endif /* -** We may need to provide the "ino_t" type. -*/ - -#ifndef INO_T_DEFINED - #define INO_T_DEFINED - typedef unsigned short ino_t; -#endif +** Data for columnar layout, collected into a single object so +** that it can be more easily passed into subroutines. +*/ +typedef struct qrfColData qrfColData; +struct qrfColData { + Qrf *p; /* The QRF instance */ + int nCol; /* Number of columns in the table */ + unsigned char bMultiRow; /* One or more cells will span multiple lines */ + unsigned char nMargin; /* Width of column margins */ + sqlite3_int64 nRow; /* Number of rows */ + sqlite3_int64 nAlloc; /* Number of cells allocated */ + sqlite3_int64 n; /* Number of cells. nCol*nRow */ + char **az; /* Content of all cells */ + int *aiWth; /* Width of each cell */ + unsigned char *abNum; /* True for each numeric cell */ + struct qrfPerCol { /* Per-column data */ + char *z; /* Cache of text for current row */ + int w; /* Computed width of this column */ + int mxW; /* Maximum natural (unwrapped) width */ + unsigned char e; /* Alignment */ + unsigned char fx; /* Width is fixed */ + unsigned char bNum; /* True if is numeric */ + } *a; /* One per column */ +}; /* -** We need to define "NAME_MAX" if it was not present in "limits.h". +** Output horizontally justified text into pOut. The text is the +** first nVal bytes of zVal. Include nWS bytes of whitespace, either +** split between both sides, or on the left, or on the right, depending +** on eAlign. */ - -#ifndef NAME_MAX -# ifdef FILENAME_MAX -# define NAME_MAX (FILENAME_MAX) -# else -# define NAME_MAX (260) -# endif -#endif +static void qrfPrintAligned( + sqlite3_str *pOut, /* Append text here */ + struct qrfPerCol *pCol, /* Information about the text to print */ + int nVal, /* Use only the first nVal bytes of zVal[] */ + int nWS /* Whitespace for horizonal alignment */ +){ + unsigned char eAlign = pCol->e & QRF_ALIGN_HMASK; + if( eAlign==QRF_Auto && pCol->bNum ) eAlign = QRF_ALIGN_Right; + if( eAlign==QRF_ALIGN_Center ){ + /* Center the text */ + sqlite3_str_appendchar(pOut, nWS/2, ' '); + qrfAppendWithTabs(pOut, pCol->z, nVal); + sqlite3_str_appendchar(pOut, nWS - nWS/2, ' '); + }else if( eAlign==QRF_ALIGN_Right ){ + /* Right justify the text */ + sqlite3_str_appendchar(pOut, nWS, ' '); + qrfAppendWithTabs(pOut, pCol->z, nVal); + }else{ + /* Left justify the text */ + qrfAppendWithTabs(pOut, pCol->z, nVal); + sqlite3_str_appendchar(pOut, nWS, ' '); + } +} /* -** We need to define "NULL_INTPTR_T" and "BAD_INTPTR_T". +** Free all the memory allocates in the qrfColData object */ - -#ifndef NULL_INTPTR_T -# define NULL_INTPTR_T ((intptr_t)(0)) -#endif - -#ifndef BAD_INTPTR_T -# define BAD_INTPTR_T ((intptr_t)(-1)) -#endif +static void qrfColDataFree(qrfColData *p){ + sqlite3_int64 i; + for(i=0; in; i++) sqlite3_free(p->az[i]); + sqlite3_free(p->az); + sqlite3_free(p->aiWth); + sqlite3_free(p->abNum); + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); +} /* -** We need to provide the necessary structures and related types. +** Allocate space for more cells in the qrfColData object. +** Return non-zero if a memory allocation fails. */ - -#ifndef DIRENT_DEFINED -#define DIRENT_DEFINED -typedef struct DIRENT DIRENT; -typedef DIRENT *LPDIRENT; -struct DIRENT { - ino_t d_ino; /* Sequence number, do not use. */ - unsigned d_attributes; /* Win32 file attributes. */ - char d_name[NAME_MAX + 1]; /* Name within the directory. */ -}; -#endif - -#ifndef DIR_DEFINED -#define DIR_DEFINED -typedef struct DIR DIR; -typedef DIR *LPDIR; -struct DIR { - intptr_t d_handle; /* Value returned by "_findfirst". */ - DIRENT d_first; /* DIRENT constructed based on "_findfirst". */ - DIRENT d_next; /* DIRENT constructed based on "_findnext". */ -}; -#endif +static int qrfColDataEnlarge(qrfColData *p){ + char **azData; + int *aiWth; + unsigned char *abNum; + p->nAlloc = 2*p->nAlloc + 10*p->nCol; + azData = sqlite3_realloc64(p->az, p->nAlloc*sizeof(char*)); + if( azData==0 ){ + qrfOom(p->p); + qrfColDataFree(p); + return 1; + } + p->az = azData; + aiWth = sqlite3_realloc64(p->aiWth, p->nAlloc*sizeof(int)); + if( aiWth==0 ){ + qrfOom(p->p); + qrfColDataFree(p); + return 1; + } + p->aiWth = aiWth; + abNum = sqlite3_realloc64(p->abNum, p->nAlloc); + if( abNum==0 ){ + qrfOom(p->p); + qrfColDataFree(p); + return 1; + } + p->abNum = abNum; + return 0; +} /* -** Provide a macro, for use by the implementation, to determine if a -** particular directory entry should be skipped over when searching for -** the next directory entry that should be returned by the readdir() or -** readdir_r() functions. +** Print a markdown or table-style row separator using ascii-art */ - -#ifndef is_filtered -# define is_filtered(a) ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) -#endif +static void qrfRowSeparator(sqlite3_str *pOut, qrfColData *p, char cSep){ + int i; + if( p->nCol>0 ){ + int useBorder = p->p->spec.bBorder!=QRF_No; + if( useBorder ){ + sqlite3_str_append(pOut, &cSep, 1); + } + sqlite3_str_appendchar(pOut, p->a[0].w+p->nMargin, '-'); + for(i=1; inCol; i++){ + sqlite3_str_append(pOut, &cSep, 1); + sqlite3_str_appendchar(pOut, p->a[i].w+p->nMargin, '-'); + } + if( useBorder ){ + sqlite3_str_append(pOut, &cSep, 1); + } + } + sqlite3_str_append(pOut, "\n", 1); +} /* -** Provide the function prototype for the POSIX compatiable getenv() -** function. This function is not thread-safe. -*/ - -extern const char *windirent_getenv(const char *name); +** UTF8 box-drawing characters. Imagine box lines like this: +** +** 1 +** | +** 4 --+-- 2 +** | +** 3 +** +** Each box characters has between 2 and 4 of the lines leading from +** the center. The characters are here identified by the numbers of +** their corresponding lines. +*/ +#define BOX_24 "\342\224\200" /* U+2500 --- */ +#define BOX_13 "\342\224\202" /* U+2502 | */ +#define BOX_23 "\342\224\214" /* U+250c ,- */ +#define BOX_34 "\342\224\220" /* U+2510 -, */ +#define BOX_12 "\342\224\224" /* U+2514 '- */ +#define BOX_14 "\342\224\230" /* U+2518 -' */ +#define BOX_123 "\342\224\234" /* U+251c |- */ +#define BOX_134 "\342\224\244" /* U+2524 -| */ +#define BOX_234 "\342\224\254" /* U+252c -,- */ +#define BOX_124 "\342\224\264" /* U+2534 -'- */ +#define BOX_1234 "\342\224\274" /* U+253c -|- */ + +/* Rounded corners: */ +#define BOX_R12 "\342\225\260" /* U+2570 '- */ +#define BOX_R23 "\342\225\255" /* U+256d ,- */ +#define BOX_R34 "\342\225\256" /* U+256e -, */ +#define BOX_R14 "\342\225\257" /* U+256f -' */ + +/* Doubled horizontal lines: */ +#define DBL_24 "\342\225\220" /* U+2550 === */ +#define DBL_123 "\342\225\236" /* U+255e |= */ +#define DBL_134 "\342\225\241" /* U+2561 =| */ +#define DBL_1234 "\342\225\252" /* U+256a =|= */ + +/* Draw horizontal line N characters long using unicode box +** characters +*/ +static void qrfBoxLine(sqlite3_str *pOut, int N, int bDbl){ + const char *azDash[2] = { + BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24, + DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 + };/* 0 1 2 3 4 5 6 7 8 9 */ + const int nDash = 30; + N *= 3; + while( N>nDash ){ + sqlite3_str_append(pOut, azDash[bDbl], nDash); + N -= nDash; + } + sqlite3_str_append(pOut, azDash[bDbl], N); +} + +/* +** Draw a horizontal separator for a QRF_STYLE_Box table. +*/ +static void qrfBoxSeparator( + sqlite3_str *pOut, + qrfColData *p, + const char *zSep1, + const char *zSep2, + const char *zSep3, + int bDbl +){ + int i; + if( p->nCol>0 ){ + int useBorder = p->p->spec.bBorder!=QRF_No; + if( useBorder ){ + sqlite3_str_appendall(pOut, zSep1); + } + qrfBoxLine(pOut, p->a[0].w+p->nMargin, bDbl); + for(i=1; inCol; i++){ + sqlite3_str_appendall(pOut, zSep2); + qrfBoxLine(pOut, p->a[i].w+p->nMargin, bDbl); + } + if( useBorder ){ + sqlite3_str_appendall(pOut, zSep3); + } + } + sqlite3_str_append(pOut, "\n", 1); +} /* -** Finally, we can provide the function prototypes for the opendir(), -** readdir(), readdir_r(), and closedir() POSIX functions. +** Load into pData the default alignment for the body of a table. */ +static void qrfLoadAlignment(qrfColData *pData, Qrf *p){ + sqlite3_int64 i; + for(i=0; inCol; i++){ + pData->a[i].e = p->spec.eDfltAlign; + if( ispec.nAlign ){ + unsigned char ax = p->spec.aAlign[i]; + if( (ax & QRF_ALIGN_HMASK)!=0 ){ + pData->a[i].e = (ax & QRF_ALIGN_HMASK) | + (pData->a[i].e & QRF_ALIGN_VMASK); + } + }else if( ispec.nWidth ){ + if( p->spec.aWidth[i]<0 ){ + pData->a[i].e = QRF_ALIGN_Right | + (pData->a[i].e & QRF_ALIGN_VMASK); + } + } + } +} -extern LPDIR opendir(const char *dirname); -extern LPDIRENT readdir(LPDIR dirp); -extern INT readdir_r(LPDIR dirp, LPDIRENT entry, LPDIRENT *result); -extern INT closedir(LPDIR dirp); - -#endif /* defined(WIN32) && defined(_MSC_VER) */ - -/************************* End test_windirent.h ********************/ -/************************* Begin test_windirent.c ******************/ /* -** 2015 November 30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** If the single column in pData->a[] with pData->n entries can be +** laid out as nCol columns with a 2-space gap between each such +** that all columns fit within nSW, then return a pointer to an array +** of integers which is the width of each column from left to right. ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** If the layout is not possible, return a NULL pointer. ** -************************************************************************* -** This file contains code to implement most of the opendir() family of -** POSIX functions on Win32 using the MSVCRT. -*/ - -#if defined(_WIN32) && defined(_MSC_VER) -/* #include "test_windirent.h" */ - -/* -** Implementation of the POSIX getenv() function using the Win32 API. -** This function is not thread-safe. +** Space to hold the returned array is from sqlite_malloc64(). */ -const char *windirent_getenv( - const char *name +static int *qrfValidLayout( + qrfColData *pData, /* Collected query results */ + Qrf *p, /* On which to report an OOM */ + int nCol, /* Attempt this many columns */ + int nSW /* Screen width */ ){ - static char value[32768]; /* Maximum length, per MSDN */ - DWORD dwSize = sizeof(value) / sizeof(char); /* Size in chars */ - DWORD dwRet; /* Value returned by GetEnvironmentVariableA() */ - - memset(value, 0, sizeof(value)); - dwRet = GetEnvironmentVariableA(name, value, dwSize); - if( dwRet==0 || dwRet>dwSize ){ - /* - ** The function call to GetEnvironmentVariableA() failed -OR- - ** the buffer is not large enough. Either way, return NULL. - */ + int i; /* Loop counter */ + int nr; /* Number of rows */ + int w = 0; /* Width of the current column */ + int t; /* Total width of all columns */ + int *aw; /* Array of individual column widths */ + + aw = sqlite3_malloc64( sizeof(int)*nCol ); + if( aw==0 ){ + qrfOom(p); + return 0; + } + nr = (pData->n + nCol - 1)/nCol; + for(i=0; in; i++){ + if( (i%nr)==0 ){ + if( i>0 ) aw[i/nr-1] = w; + w = pData->aiWth[i]; + }else if( pData->aiWth[i]>w ){ + w = pData->aiWth[i]; + } + } + aw[nCol-1] = w; + for(t=i=0; inSW ){ + sqlite3_free(aw); return 0; + } + return aw; +} + +/* +** The output is single-column and the bSplitColumn flag is set. +** Check to see if the single-column output can be split into multiple +** columns that appear side-by-side. Adjust pData appropriately. +*/ +static void qrfSplitColumn(qrfColData *pData, Qrf *p){ + int nCol = 1; + int *aw = 0; + char **az = 0; + int *aiWth = 0; + unsigned char *abNum = 0; + int nColNext = 2; + int w; + struct qrfPerCol *a = 0; + sqlite3_int64 nRow = 1; + sqlite3_int64 i; + while( 1/*exit-by-break*/ ){ + int *awNew = qrfValidLayout(pData, p, nColNext, p->spec.nScreenWidth); + if( awNew==0 ) break; + sqlite3_free(aw); + aw = awNew; + nCol = nColNext; + nRow = (pData->n + nCol - 1)/nCol; + if( nRow==1 ) break; + nColNext++; + while( (pData->n + nColNext - 1)/nColNext == nRow ) nColNext++; + } + if( nCol==1 ){ + sqlite3_free(aw); + return; /* Cannot do better than 1 column */ + } + az = sqlite3_malloc64( nRow*nCol*sizeof(char*) ); + if( az==0 ){ + qrfOom(p); + return; + } + aiWth = sqlite3_malloc64( nRow*nCol*sizeof(int) ); + if( aiWth==0 ){ + sqlite3_free(az); + qrfOom(p); + return; + } + a = sqlite3_malloc64( nCol*sizeof(struct qrfPerCol) ); + if( a==0 ){ + sqlite3_free(az); + sqlite3_free(aiWth); + qrfOom(p); + return; + } + abNum = sqlite3_malloc64( nRow*nCol ); + if( abNum==0 ){ + sqlite3_free(az); + sqlite3_free(aiWth); + sqlite3_free(a); + qrfOom(p); + return; + } + for(i=0; in; i++){ + sqlite3_int64 j = (i%nRow)*nCol + (i/nRow); + az[j] = pData->az[i]; + abNum[j]= pData->abNum[i]; + pData->az[i] = 0; + aiWth[j] = pData->aiWth[i]; + } + while( ia[0].e; + } + sqlite3_free(pData->az); + sqlite3_free(pData->aiWth); + sqlite3_free(pData->a); + sqlite3_free(pData->abNum); + sqlite3_free(aw); + pData->az = az; + pData->aiWth = aiWth; + pData->a = a; + pData->abNum = abNum; + pData->nCol = nCol; + pData->n = pData->nAlloc = nRow*nCol; + for(i=w=0; inMargin = (p->spec.nScreenWidth - w)/(nCol - 1); + if( pData->nMargin>5 ) pData->nMargin = 5; +} + +/* +** Adjust the layout for the screen width restriction +*/ +static void qrfRestrictScreenWidth(qrfColData *pData, Qrf *p){ + int sepW; /* Width of all box separators and margins */ + int sumW; /* Total width of data area over all columns */ + int targetW; /* Desired total data area */ + int i; /* Loop counters */ + int nCol; /* Number of columns */ + + pData->nMargin = 2; /* Default to normal margins */ + if( p->spec.nScreenWidth==0 ) return; + if( p->spec.eStyle==QRF_STYLE_Column ){ + sepW = pData->nCol*2 - 2; }else{ - /* - ** The function call to GetEnvironmentVariableA() succeeded - ** -AND- the buffer contains the entire value. - */ - return value; + sepW = pData->nCol*3 + 1; + if( p->spec.bBorder==QRF_No ) sepW -= 2; + } + nCol = pData->nCol; + for(i=sumW=0; ia[i].w; + if( p->spec.nScreenWidth >= sumW+sepW ) return; + + /* First thing to do is reduce the separation between columns */ + pData->nMargin = 0; + if( p->spec.eStyle==QRF_STYLE_Column ){ + sepW = pData->nCol - 1; + }else{ + sepW = pData->nCol + 1; + if( p->spec.bBorder==QRF_No ) sepW -= 2; + } + targetW = p->spec.nScreenWidth - sepW; + +#define MIN_SQUOZE 8 +#define MIN_EX_SQUOZE 16 + /* Reduce the width of the widest eligible column. A column is + ** eligible for narrowing if: + ** + ** * It is not a fixed-width column (a[0].fx is false) + ** * The current width is more than MIN_SQUOZE + ** * Either: + ** + The current width is more then MIN_EX_SQUOZE, or + ** + The current width is more than half the max width (a[].mxW) + ** + ** Keep making reductions until either no more reductions are + ** possible or until the size target is reached. + */ + while( sumW > targetW ){ + int gain, w; + int ix = -1; + int mx = 0; + for(i=0; ia[i].fx==0 + && (w = pData->a[i].w)>mx + && w>MIN_SQUOZE + && (w>MIN_EX_SQUOZE || w*2>pData->a[i].mxW) + ){ + ix = i; + mx = w; + } + } + if( ix<0 ) break; + if( mx>=MIN_SQUOZE*2 ){ + gain = mx/2; + }else{ + gain = mx - MIN_SQUOZE; + } + if( sumW - gain < targetW ){ + gain = sumW - targetW; + } + sumW -= gain; + pData->a[ix].w -= gain; + pData->bMultiRow = 1; } } /* -** Implementation of the POSIX opendir() function using the MSVCRT. +** Columnar modes require that the entire query be evaluated first, with +** results written into memory, so that we can compute appropriate column +** widths. */ -LPDIR opendir( - const char *dirname -){ - struct _finddata_t data; - LPDIR dirp = (LPDIR)sqlite3_malloc(sizeof(DIR)); - SIZE_T namesize = sizeof(data.name) / sizeof(data.name[0]); +static void qrfColumnar(Qrf *p){ + sqlite3_int64 i, j; /* Loop counters */ + const char *colSep = 0; /* Column separator text */ + const char *rowSep = 0; /* Row terminator text */ + const char *rowStart = 0; /* Row start text */ + int szColSep, szRowSep, szRowStart; /* Size in bytes of previous 3 */ + int rc; /* Result code */ + int nColumn = p->nCol; /* Number of columns */ + int bWW; /* True to do word-wrap */ + sqlite3_str *pStr; /* Temporary rendering */ + qrfColData data; /* Columnar layout data */ + int bRTrim; /* Trim trailing space */ - if( dirp==NULL ) return NULL; - memset(dirp, 0, sizeof(DIR)); + rc = sqlite3_step(p->pStmt); + if( rc!=SQLITE_ROW || nColumn==0 ){ + return; /* No output */ + } - /* TODO: Remove this if Unix-style root paths are not used. */ - if( sqlite3_stricmp(dirname, "/")==0 ){ - dirname = windirent_getenv("SystemDrive"); + /* Initialize the data container */ + memset(&data, 0, sizeof(data)); + data.nCol = p->nCol; + data.p = p; + data.a = sqlite3_malloc64( nColumn*sizeof(struct qrfPerCol) ); + if( data.a==0 ){ + qrfOom(p); + return; + } + memset(data.a, 0, nColumn*sizeof(struct qrfPerCol) ); + if( qrfColDataEnlarge(&data) ) return; + assert( data.az!=0 ); + + /* Load the column header names and all cell content into data */ + if( p->spec.bTitles==QRF_Yes ){ + unsigned char saved_eText = p->spec.eText; + p->spec.eText = p->spec.eTitle; + memset(data.abNum, 0, nColumn); + for(i=0; ipStmt,i); + int nNL = 0; + int n, w; + pStr = sqlite3_str_new(p->db); + qrfEncodeText(p, pStr, z ? z : ""); + n = sqlite3_str_length(pStr); + qrfStrErr(p, pStr); + z = data.az[data.n] = sqlite3_str_finish(pStr); + if( p->spec.nTitleLimit ){ + nNL = 0; + data.aiWth[data.n] = w = qrfTitleLimit(data.az[data.n], + p->spec.nTitleLimit ); + }else{ + data.aiWth[data.n] = w = qrfDisplayWidth(z, n, &nNL); + } + data.n++; + if( w>data.a[i].mxW ) data.a[i].mxW = w; + if( nNL ) data.bMultiRow = 1; + } + p->spec.eText = saved_eText; + p->nRow++; + } + do{ + if( data.n+nColumn > data.nAlloc ){ + if( qrfColDataEnlarge(&data) ) return; + } + for(i=0; ipStmt,i); + pStr = sqlite3_str_new(p->db); + qrfRenderValue(p, pStr, i); + n = sqlite3_str_length(pStr); + qrfStrErr(p, pStr); + z = data.az[data.n] = sqlite3_str_finish(pStr); + data.abNum[data.n] = eType==SQLITE_INTEGER || eType==SQLITE_FLOAT; + data.aiWth[data.n] = w = qrfDisplayWidth(z, n, &nNL); + data.n++; + if( w>data.a[i].mxW ) data.a[i].mxW = w; + if( nNL ) data.bMultiRow = 1; + } + p->nRow++; + }while( sqlite3_step(p->pStmt)==SQLITE_ROW && p->iErr==SQLITE_OK ); + if( p->iErr ){ + qrfColDataFree(&data); + return; } - memset(&data, 0, sizeof(struct _finddata_t)); - _snprintf(data.name, namesize, "%s\\*", dirname); - dirp->d_handle = _findfirst(data.name, &data); + /* Compute the width and alignment of every column */ + if( p->spec.bTitles==QRF_No ){ + qrfLoadAlignment(&data, p); + }else{ + unsigned char e; + if( p->spec.eTitleAlign==QRF_Auto ){ + e = QRF_ALIGN_Center; + }else{ + e = p->spec.eTitleAlign; + } + for(i=0; id_handle==BAD_INTPTR_T ){ - closedir(dirp); - return NULL; + for(i=0; ispec.nWidth ){ + w = p->spec.aWidth[i]; + if( w==(-32768) ){ + w = 0; + if( p->spec.nAlign>i && (p->spec.aAlign[i] & QRF_ALIGN_HMASK)==0 ){ + data.a[i].e |= QRF_ALIGN_Right; + } + }else if( w<0 ){ + w = -w; + if( p->spec.nAlign>i && (p->spec.aAlign[i] & QRF_ALIGN_HMASK)==0 ){ + data.a[i].e |= QRF_ALIGN_Right; + } + } + if( w ) data.a[i].fx = 1; + } + if( w==0 ){ + w = data.a[i].mxW; + if( p->spec.nWrap>0 && w>p->spec.nWrap ){ + w = p->spec.nWrap; + data.bMultiRow = 1; + } + }else if( (data.bMultiRow==0 || w==1) && data.a[i].mxW>w ){ + data.bMultiRow = 1; + if( w==1 ){ + /* If aiWth[j] is 2 or more, then there might be a double-wide + ** character somewhere. So make the column width at least 2. */ + w = 2; + } + } + data.a[i].w = w; + } + + if( nColumn==1 + && data.n>1 + && p->spec.bSplitColumn==QRF_Yes + && p->spec.eStyle==QRF_STYLE_Column + && p->spec.bTitles==QRF_No + && p->spec.nScreenWidth>data.a[0].w+3 + ){ + /* Attempt to convert single-column tables into multi-column by + ** verticle wrapping, if the screen is wide enough and if the + ** bSplitColumn flag is set. */ + qrfSplitColumn(&data, p); + nColumn = data.nCol; + }else{ + /* Adjust the column widths due to screen width restrictions */ + qrfRestrictScreenWidth(&data, p); + } + + /* Draw the line across the top of the table. Also initialize + ** the row boundary and column separator texts. */ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + if( data.nMargin ){ + rowStart = BOX_13 " "; + colSep = " " BOX_13 " "; + rowSep = " " BOX_13 "\n"; + }else{ + rowStart = BOX_13; + colSep = BOX_13; + rowSep = BOX_13 "\n"; + } + if( p->spec.bBorder==QRF_No){ + rowStart += 3; + rowSep = "\n"; + }else{ + qrfBoxSeparator(p->pOut, &data, BOX_R23, BOX_234, BOX_R34, 0); + } + break; + case QRF_STYLE_Table: + if( data.nMargin ){ + rowStart = "| "; + colSep = " | "; + rowSep = " |\n"; + }else{ + rowStart = "|"; + colSep = "|"; + rowSep = "|\n"; + } + if( p->spec.bBorder==QRF_No ){ + rowStart += 1; + rowSep = "\n"; + }else{ + qrfRowSeparator(p->pOut, &data, '+'); + } + break; + case QRF_STYLE_Column: { + static const char zSpace[] = " "; + rowStart = ""; + if( data.nMargin<2 ){ + colSep = " "; + }else if( data.nMargin<=5 ){ + colSep = &zSpace[5-data.nMargin]; + }else{ + colSep = zSpace; + } + rowSep = "\n"; + break; + } + default: /*case QRF_STYLE_Markdown:*/ + if( data.nMargin ){ + rowStart = "| "; + colSep = " | "; + rowSep = " |\n"; + }else{ + rowStart = "|"; + colSep = "|"; + rowSep = "|\n"; + } + break; } + szRowStart = (int)strlen(rowStart); + szRowSep = (int)strlen(rowSep); + szColSep = (int)strlen(colSep); - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ){ -next: + bWW = (p->spec.bWordWrap==QRF_Yes && data.bMultiRow); + if( p->spec.eStyle==QRF_STYLE_Column + || (p->spec.bBorder==QRF_No + && (p->spec.eStyle==QRF_STYLE_Box || p->spec.eStyle==QRF_STYLE_Table) + ) + ){ + bRTrim = 1; + }else{ + bRTrim = 0; + } + for(i=0; ipOut)==SQLITE_OK; i+=nColumn){ + int bMore; + int nRow = 0; - memset(&data, 0, sizeof(struct _finddata_t)); - if( _findnext(dirp->d_handle, &data)==-1 ){ - closedir(dirp); - return NULL; + /* Draw a single row of the table. This might be the title line + ** (if there is a title line) or a row in the body of the table. + ** The column number will be j. The row number is i/nColumn. + */ + for(j=0; jpOut, rowStart, szRowStart); + bMore = 0; + for(j=0; jpOut, &data.a[j], nThis, nWS); + data.a[j].z += iNext; + if( data.a[j].z[0]!=0 ){ + bMore = 1; + } + if( jpOut, colSep, szColSep); + }else{ + if( bRTrim ) qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, rowSep, szRowSep); + } + } + }while( bMore && ++nRow < p->mxHeight ); + if( bMore ){ + /* This row was terminated by nLineLimit. Show ellipsis. */ + sqlite3_str_append(p->pOut, rowStart, szRowStart); + for(j=0; jpOut, data.a[j].w, ' '); + }else{ + int nE = 3; + if( nE>data.a[j].w ) nE = data.a[j].w; + data.a[j].z = "..."; + qrfPrintAligned(p->pOut, &data.a[j], nE, data.a[j].w-nE); + } + if( jpOut, colSep, szColSep); + }else{ + if( bRTrim ) qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, rowSep, szRowSep); + } + } } - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; + /* Draw either (1) the separator between the title line and the body + ** of the table, or (2) separators between individual rows of the table + ** body. isTitleDataSeparator will be true if we are doing (1). + */ + if( (i==0 || data.bMultiRow) && i+nColumnspec.bTitles==QRF_Yes); + if( isTitleDataSeparator ){ + qrfLoadAlignment(&data, p); + } + switch( p->spec.eStyle ){ + case QRF_STYLE_Table: { + if( isTitleDataSeparator || data.bMultiRow ){ + qrfRowSeparator(p->pOut, &data, '+'); + } + break; + } + case QRF_STYLE_Box: { + if( isTitleDataSeparator ){ + qrfBoxSeparator(p->pOut, &data, DBL_123, DBL_1234, DBL_134, 1); + }else if( data.bMultiRow ){ + qrfBoxSeparator(p->pOut, &data, BOX_123, BOX_1234, BOX_134, 0); + } + break; + } + case QRF_STYLE_Markdown: { + if( isTitleDataSeparator ){ + qrfRowSeparator(p->pOut, &data, '|'); + } + break; + } + case QRF_STYLE_Column: { + if( isTitleDataSeparator ){ + for(j=0; jpOut, data.a[j].w, '-'); + if( jpOut, colSep, szColSep); + }else{ + qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, rowSep, szRowSep); + } + } + }else if( data.bMultiRow ){ + qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, "\n", 1); + } + break; + } + } + } } - dirp->d_first.d_attributes = data.attrib; - strncpy(dirp->d_first.d_name, data.name, NAME_MAX); - dirp->d_first.d_name[NAME_MAX] = '\0'; + /* Draw the line across the bottom of the table */ + if( p->spec.bBorder!=QRF_No ){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + qrfBoxSeparator(p->pOut, &data, BOX_R12, BOX_124, BOX_R14, 0); + break; + case QRF_STYLE_Table: + qrfRowSeparator(p->pOut, &data, '+'); + break; + } + } + qrfWrite(p); - return dirp; + qrfColDataFree(&data); + return; } /* -** Implementation of the POSIX readdir() function using the MSVCRT. +** Parameter azArray points to a zero-terminated array of strings. zStr +** points to a single nul-terminated string. Return non-zero if zStr +** is equal, according to strcmp(), to any of the strings in the array. +** Otherwise, return zero. */ -LPDIRENT readdir( - LPDIR dirp -){ - struct _finddata_t data; - - if( dirp==NULL ) return NULL; - - if( dirp->d_first.d_ino==0 ){ - dirp->d_first.d_ino++; - dirp->d_next.d_ino++; - - return &dirp->d_first; +static int qrfStringInArray(const char *zStr, const char **azArray){ + int i; + if( zStr==0 ) return 0; + for(i=0; azArray[i]; i++){ + if( 0==strcmp(zStr, azArray[i]) ) return 1; } - -next: - - memset(&data, 0, sizeof(struct _finddata_t)); - if( _findnext(dirp->d_handle, &data)==-1 ) return NULL; - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; - - dirp->d_next.d_ino++; - dirp->d_next.d_attributes = data.attrib; - strncpy(dirp->d_next.d_name, data.name, NAME_MAX); - dirp->d_next.d_name[NAME_MAX] = '\0'; - - return &dirp->d_next; + return 0; } /* -** Implementation of the POSIX readdir_r() function using the MSVCRT. +** Print out an EXPLAIN with indentation. This is a two-pass algorithm. +** +** On the first pass, we compute aiIndent[iOp] which is the amount of +** indentation to apply to the iOp-th opcode. The output actually occurs +** on the second pass. +** +** The indenting rules are: +** +** * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent +** all opcodes that occur between the p2 jump destination and the opcode +** itself by 2 spaces. +** +** * Do the previous for "Return" instructions for when P2 is positive. +** See tag-20220407a in wherecode.c and vdbe.c. +** +** * For each "Goto", if the jump destination is earlier in the program +** and ends on one of: +** Yield SeekGt SeekLt RowSetRead Rewind +** or if the P1 parameter is one instead of zero, +** then indent all opcodes between the earlier instruction +** and "Goto" by 2 spaces. */ -INT readdir_r( - LPDIR dirp, - LPDIRENT entry, - LPDIRENT *result -){ - struct _finddata_t data; +static void qrfExplain(Qrf *p){ + int *abYield = 0; /* abYield[iOp] is rue if opcode iOp is an OP_Yield */ + int *aiIndent = 0; /* Indent the iOp-th opcode by aiIndent[iOp] */ + i64 nAlloc = 0; /* Allocated size of aiIndent[], abYield */ + int nIndent = 0; /* Number of entries in aiIndent[] */ + int iOp; /* Opcode number */ + int i; /* Column loop counter */ - if( dirp==NULL ) return EBADF; + const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", + "Return", 0 }; + const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", + "Rewind", 0 }; + const char *azGoto[] = { "Goto", 0 }; - if( dirp->d_first.d_ino==0 ){ - dirp->d_first.d_ino++; - dirp->d_next.d_ino++; + /* The caller guarantees that the leftmost 4 columns of the statement + ** passed to this function are equivalent to the leftmost 4 columns + ** of EXPLAIN statement output. In practice the statement may be + ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ + assert( sqlite3_column_count(p->pStmt)>=4 ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 0), "addr" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 1), "opcode" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 2), "p1" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 3), "p2" ) ); + + for(iOp=0; SQLITE_ROW==sqlite3_step(p->pStmt) && !p->iErr; iOp++){ + int iAddr = sqlite3_column_int(p->pStmt, 0); + const char *zOp = (const char*)sqlite3_column_text(p->pStmt, 1); + int p1 = sqlite3_column_int(p->pStmt, 2); + int p2 = sqlite3_column_int(p->pStmt, 3); + + /* Assuming that p2 is an instruction address, set variable p2op to the + ** index of that instruction in the aiIndent[] array. p2 and p2op may be + ** different if the current instruction is part of a sub-program generated + ** by an SQL trigger or foreign key. */ + int p2op = (p2 + (iOp-iAddr)); - entry->d_ino = dirp->d_first.d_ino; - entry->d_attributes = dirp->d_first.d_attributes; - strncpy(entry->d_name, dirp->d_first.d_name, NAME_MAX); - entry->d_name[NAME_MAX] = '\0'; + /* Grow the aiIndent array as required */ + if( iOp>=nAlloc ){ + nAlloc += 100; + aiIndent = (int*)sqlite3_realloc64(aiIndent, nAlloc*sizeof(int)); + abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); + if( aiIndent==0 || abYield==0 ){ + qrfOom(p); + sqlite3_free(aiIndent); + sqlite3_free(abYield); + return; + } + } - *result = entry; - return 0; + abYield[iOp] = qrfStringInArray(zOp, azYield); + aiIndent[iOp] = 0; + nIndent = iOp+1; + if( qrfStringInArray(zOp, azNext) && p2op>0 ){ + for(i=p2op; id_handle, &data)==-1 ){ - *result = NULL; - return ENOENT; + /* Second pass. Actually generate output */ + sqlite3_reset(p->pStmt); + if( p->iErr==SQLITE_OK ){ + static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; + static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; + static const int aScanExpWidth[] = {4,15, 6, 13, 4, 4, 4, 13, 2, 13}; + static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; + const int *aWidth = aExplainWidth; + const int *aMap = aExplainMap; + int nWidth = sizeof(aExplainWidth)/sizeof(int); + int iIndent = 1; + int nArg = p->nCol; + if( p->spec.eStyle==QRF_STYLE_StatsVm ){ + aWidth = aScanExpWidth; + aMap = aScanExpMap; + nWidth = sizeof(aScanExpWidth)/sizeof(int); + iIndent = 3; + } + if( nArg>nWidth ) nArg = nWidth; + + for(iOp=0; sqlite3_step(p->pStmt)==SQLITE_ROW && !p->iErr; iOp++){ + /* If this is the first row seen, print out the headers */ + if( iOp==0 ){ + for(i=0; ipStmt, aMap[i]); + qrfWidthPrint(p,p->pOut, aWidth[i], zCol); + if( i==nArg-1 ){ + sqlite3_str_append(p->pOut, "\n", 1); + }else{ + sqlite3_str_append(p->pOut, " ", 2); + } + } + for(i=0; ipOut, "%.*c", aWidth[i], '-'); + if( i==nArg-1 ){ + sqlite3_str_append(p->pOut, "\n", 1); + }else{ + sqlite3_str_append(p->pOut, " ", 2); + } + } + } + + for(i=0; ipStmt, aMap[i]); + int len; + if( i==nArg-1 ) w = 0; + if( zVal==0 ) zVal = ""; + len = (int)sqlite3_qrf_wcswidth(zVal); + if( len>w ){ + w = len; + zSep = " "; + } + if( i==iIndent && aiIndent && iOppOut, aiIndent[iOp], ' '); + } + qrfWidthPrint(p, p->pOut, w, zVal); + if( i==nArg-1 ){ + sqlite3_str_append(p->pOut, "\n", 1); + }else{ + sqlite3_str_appendall(p->pOut, zSep); + } + } + p->nRow++; + } + qrfWrite(p); } + sqlite3_free(aiIndent); +} - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; - - entry->d_ino = (ino_t)-1; /* not available */ - entry->d_attributes = data.attrib; - strncpy(entry->d_name, data.name, NAME_MAX); - entry->d_name[NAME_MAX] = '\0'; - - *result = entry; - return 0; +/* +** Do a "scanstatus vm" style EXPLAIN listing on p->pStmt. +** +** p->pStmt is probably not an EXPLAIN query. Instead, construct a +** new query that is a bytecode() rendering of p->pStmt with extra +** columns for the "scanstatus vm" outputs, and run the results of +** that new query through the normal EXPLAIN formatting. +*/ +static void qrfScanStatusVm(Qrf *p){ + sqlite3_stmt *pOrigStmt = p->pStmt; + sqlite3_stmt *pExplain; + int rc; + static const char *zSql = + " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," + " format('% 6s (%.2f%%)'," + " CASE WHEN ncycle<100_000 THEN ncycle || ' '" + " WHEN ncycle<100_000_000 THEN (ncycle/1_000) || 'K'" + " WHEN ncycle<100_000_000_000 THEN (ncycle/1_000_000) || 'M'" + " ELSE (ncycle/1000_000_000) || 'G' END," + " ncycle*100.0/(sum(ncycle) OVER ())" + " ) AS cycles" + " FROM bytecode(?1)"; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pExplain, 0); + if( rc ){ + qrfError(p, rc, "%s", sqlite3_errmsg(p->db)); + sqlite3_finalize(pExplain); + return; + } + sqlite3_bind_pointer(pExplain, 1, pOrigStmt, "stmt-pointer", 0); + p->pStmt = pExplain; + p->nCol = 10; + qrfExplain(p); + sqlite3_finalize(pExplain); + p->pStmt = pOrigStmt; } /* -** Implementation of the POSIX closedir() function using the MSVCRT. +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return 1 if quoting is required. Return 0 if no quoting is required. */ -INT closedir( - LPDIR dirp -){ - INT result = 0; - if( dirp==NULL ) return EINVAL; - - if( dirp->d_handle!=NULL_INTPTR_T && dirp->d_handle!=BAD_INTPTR_T ){ - result = _findclose(dirp->d_handle); +static int qrf_need_quote(const char *zName){ + int i; + const unsigned char *z = (const unsigned char*)zName; + if( z==0 ) return 1; + if( !qrfAlpha(z[0]) ) return 1; + for(i=0; z[i]; i++){ + if( !qrfAlnum(z[i]) ) return 1; } - - sqlite3_free(dirp); - return result; + return sqlite3_keyword_check(zName, i)!=0; } -#endif /* defined(WIN32) && defined(_MSC_VER) */ - -/************************* End test_windirent.c ********************/ -#define dirent DIRENT -#endif -/************************* Begin ../ext/misc/shathree.c ******************/ /* -** 2017-03-08 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This SQLite extension implements functions that compute SHA3 hashes. -** Two SQL functions are implemented: -** -** sha3(X,SIZE) -** sha3_query(Y,SIZE) -** -** The sha3(X) function computes the SHA3 hash of the input X, or NULL if -** X is NULL. -** -** The sha3_query(Y) function evalutes all queries in the SQL statements of Y -** and returns a hash of their results. -** -** The SIZE argument is optional. If omitted, the SHA3-256 hash algorithm -** is used. If SIZE is included it must be one of the integers 224, 256, -** 384, or 512, to determine SHA3 hash variant that is computed. +** Helper function for QRF_STYLE_Json and QRF_STYLE_JObject. +** The initial "{" for a JSON object that will contain row content +** has been output. Now output all the content. */ -/* #include "sqlite3ext.h" */ -SQLITE_EXTENSION_INIT1 -#include -#include -#include -/* typedef sqlite3_uint64 u64; */ +static void qrfOneJsonRow(Qrf *p){ + int i, nItem; + for(nItem=i=0; inCol; i++){ + const char *zCName; + zCName = sqlite3_column_name(p->pStmt, i); + if( nItem>0 ) sqlite3_str_append(p->pOut, ",", 1); + nItem++; + qrfEncodeText(p, p->pOut, zCName); + sqlite3_str_append(p->pOut, ":", 1); + qrfRenderValue(p, p->pOut, i); + } + qrfWrite(p); +} -/****************************************************************************** -** The Hash Engine -*/ /* -** Macros to determine whether the machine is big or little endian, -** and whether or not that determination is run-time or compile-time. -** -** For best performance, an attempt is made to guess at the byte-order -** using C-preprocessor macros. If that is unsuccessful, or if -** -DSHA3_BYTEORDER=0 is set, then byte-order is determined -** at run-time. +** Render a single row of output for non-columnar styles - any +** style that lets us render row by row as the content is received +** from the query. */ -#ifndef SHA3_BYTEORDER -# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__) -# define SHA3_BYTEORDER 1234 -# elif defined(sparc) || defined(__ppc__) -# define SHA3_BYTEORDER 4321 -# else -# define SHA3_BYTEORDER 0 -# endif -#endif - +static void qrfOneSimpleRow(Qrf *p){ + int i; + switch( p->spec.eStyle ){ + case QRF_STYLE_Off: + case QRF_STYLE_Count: { + /* No-op */ + break; + } + case QRF_STYLE_Json: { + if( p->nRow==0 ){ + sqlite3_str_append(p->pOut, "[{", 2); + }else{ + sqlite3_str_append(p->pOut, "},\n{", 4); + } + qrfOneJsonRow(p); + break; + } + case QRF_STYLE_JObject: { + if( p->nRow==0 ){ + sqlite3_str_append(p->pOut, "{", 1); + }else{ + sqlite3_str_append(p->pOut, "}\n{", 3); + } + qrfOneJsonRow(p); + break; + } + case QRF_STYLE_Html: { + if( p->nRow==0 && p->spec.bTitles==QRF_Yes ){ + sqlite3_str_append(p->pOut, "", 4); + for(i=0; inCol; i++){ + const char *zCName = sqlite3_column_name(p->pStmt, i); + sqlite3_str_append(p->pOut, "\n", 5); + qrfEncodeText(p, p->pOut, zCName); + } + sqlite3_str_append(p->pOut, "\n\n", 7); + } + sqlite3_str_append(p->pOut, "", 4); + for(i=0; inCol; i++){ + sqlite3_str_append(p->pOut, "\n", 5); + qrfRenderValue(p, p->pOut, i); + } + sqlite3_str_append(p->pOut, "\n\n", 7); + qrfWrite(p); + break; + } + case QRF_STYLE_Insert: { + unsigned int mxIns = p->spec.nMultiInsert; + int szStart = sqlite3_str_length(p->pOut); + if( p->u.nIns==0 || p->u.nIns>=mxIns ){ + if( p->u.nIns ){ + sqlite3_str_append(p->pOut, ";\n", 2); + p->u.nIns = 0; + } + if( qrf_need_quote(p->spec.zTableName) ){ + sqlite3_str_appendf(p->pOut,"INSERT INTO \"%w\"",p->spec.zTableName); + }else{ + sqlite3_str_appendf(p->pOut,"INSERT INTO %s",p->spec.zTableName); + } + if( p->spec.bTitles==QRF_Yes ){ + for(i=0; inCol; i++){ + const char *zCName = sqlite3_column_name(p->pStmt, i); + if( qrf_need_quote(zCName) ){ + sqlite3_str_appendf(p->pOut, "%c\"%w\"", + i==0 ? '(' : ',', zCName); + }else{ + sqlite3_str_appendf(p->pOut, "%c%s", + i==0 ? '(' : ',', zCName); + } + } + sqlite3_str_append(p->pOut, ")", 1); + } + sqlite3_str_append(p->pOut," VALUES(", 8); + }else{ + sqlite3_str_append(p->pOut,",\n (", 5); + } + for(i=0; inCol; i++){ + if( i>0 ) sqlite3_str_append(p->pOut, ",", 1); + qrfRenderValue(p, p->pOut, i); + } + p->u.nIns += sqlite3_str_length(p->pOut) + 2 - szStart; + if( p->u.nIns>=mxIns ){ + sqlite3_str_append(p->pOut, ");\n", 3); + p->u.nIns = 0; + }else{ + sqlite3_str_append(p->pOut, ")", 1); + } + qrfWrite(p); + break; + } + case QRF_STYLE_Line: { + sqlite3_str *pVal; + int mxW; + int bWW; + int nSep; + if( p->u.sLine.azCol==0 ){ + p->u.sLine.azCol = sqlite3_malloc64( p->nCol*sizeof(char*) ); + if( p->u.sLine.azCol==0 ){ + qrfOom(p); + break; + } + p->u.sLine.mxColWth = 0; + for(i=0; inCol; i++){ + int sz; + const char *zCName = sqlite3_column_name(p->pStmt, i); + if( zCName==0 ) zCName = "unknown"; + p->u.sLine.azCol[i] = sqlite3_mprintf("%s", zCName); + if( p->spec.nTitleLimit>0 ){ + (void)qrfTitleLimit(p->u.sLine.azCol[i], p->spec.nTitleLimit); + } + sz = (int)sqlite3_qrf_wcswidth(p->u.sLine.azCol[i]); + if( sz > p->u.sLine.mxColWth ) p->u.sLine.mxColWth = sz; + } + } + if( p->nRow ) sqlite3_str_append(p->pOut, "\n", 1); + pVal = sqlite3_str_new(p->db); + nSep = (int)strlen(p->spec.zColumnSep); + mxW = p->mxWidth - (nSep + p->u.sLine.mxColWth); + bWW = p->spec.bWordWrap==QRF_Yes; + for(i=0; inCol; i++){ + const char *zVal; + int cnt = 0; + qrfWidthPrint(p, p->pOut, -p->u.sLine.mxColWth, p->u.sLine.azCol[i]); + sqlite3_str_append(p->pOut, p->spec.zColumnSep, nSep); + qrfRenderValue(p, pVal, i); + zVal = sqlite3_str_value(pVal); + if( zVal==0 ) zVal = ""; + do{ + int nThis, nWide, iNext; + qrfWrapLine(zVal, mxW, bWW, &nThis, &nWide, &iNext); + if( cnt ){ + sqlite3_str_appendchar(p->pOut,p->u.sLine.mxColWth+nSep,' '); + } + cnt++; + if( cnt>p->mxHeight ){ + zVal = "..."; + nThis = iNext = 3; + } + sqlite3_str_append(p->pOut, zVal, nThis); + sqlite3_str_append(p->pOut, "\n", 1); + zVal += iNext; + }while( zVal[0] ); + sqlite3_str_reset(pVal); + } + qrfStrErr(p, pVal); + sqlite3_free(sqlite3_str_finish(pVal)); + qrfWrite(p); + break; + } + case QRF_STYLE_Eqp: { + const char *zEqpLine = (const char*)sqlite3_column_text(p->pStmt,3); + int iEqpId = sqlite3_column_int(p->pStmt, 0); + int iParentId = sqlite3_column_int(p->pStmt, 1); + if( zEqpLine==0 ) zEqpLine = ""; + if( zEqpLine[0]=='-' ) qrfEqpRender(p, 0); + qrfEqpAppend(p, iEqpId, iParentId, zEqpLine); + break; + } + default: { /* QRF_STYLE_List */ + if( p->nRow==0 && p->spec.bTitles==QRF_Yes ){ + int saved_eText = p->spec.eText; + p->spec.eText = p->spec.eTitle; + for(i=0; inCol; i++){ + const char *zCName = sqlite3_column_name(p->pStmt, i); + if( i>0 ) sqlite3_str_appendall(p->pOut, p->spec.zColumnSep); + qrfEncodeText(p, p->pOut, zCName); + } + sqlite3_str_appendall(p->pOut, p->spec.zRowSep); + qrfWrite(p); + p->spec.eText = saved_eText; + } + for(i=0; inCol; i++){ + if( i>0 ) sqlite3_str_appendall(p->pOut, p->spec.zColumnSep); + qrfRenderValue(p, p->pOut, i); + } + sqlite3_str_appendall(p->pOut, p->spec.zRowSep); + qrfWrite(p); + break; + } + } + p->nRow++; +} /* -** State structure for a SHA3 hash in progress +** Initialize the internal Qrf object. */ -typedef struct SHA3Context SHA3Context; -struct SHA3Context { - union { - u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ - unsigned char x[1600]; /* ... or 1600 bytes */ - } u; - unsigned nRate; /* Bytes of input accepted per Keccak iteration */ - unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ - unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ -}; +static void qrfInitialize( + Qrf *p, /* State object to be initialized */ + sqlite3_stmt *pStmt, /* Query whose output to be formatted */ + const sqlite3_qrf_spec *pSpec, /* Format specification */ + char **pzErr /* Write errors here */ +){ + size_t sz; /* Size of pSpec[], based on pSpec->iVersion */ + memset(p, 0, sizeof(*p)); + p->pzErr = pzErr; + if( pSpec->iVersion>1 ){ + qrfError(p, SQLITE_ERROR, + "unusable sqlite3_qrf_spec.iVersion (%d)", + pSpec->iVersion); + return; + } + p->pStmt = pStmt; + p->db = sqlite3_db_handle(pStmt); + p->pOut = sqlite3_str_new(p->db); + if( p->pOut==0 ){ + qrfOom(p); + return; + } + p->iErr = SQLITE_OK; + p->nCol = sqlite3_column_count(p->pStmt); + p->nRow = 0; + sz = sizeof(sqlite3_qrf_spec); + memcpy(&p->spec, pSpec, sz); + if( p->spec.zNull==0 ) p->spec.zNull = ""; + p->mxWidth = p->spec.nScreenWidth; + if( p->mxWidth<=0 ) p->mxWidth = QRF_MAX_WIDTH; + p->mxHeight = p->spec.nLineLimit; + if( p->mxHeight<=0 ) p->mxHeight = 2147483647; + if( p->spec.eStyle>QRF_STYLE_Table ) p->spec.eStyle = QRF_Auto; + if( p->spec.eEsc>QRF_ESC_Symbol ) p->spec.eEsc = QRF_Auto; + if( p->spec.eText>QRF_TEXT_Relaxed ) p->spec.eText = QRF_Auto; + if( p->spec.eTitle>QRF_TEXT_Relaxed ) p->spec.eTitle = QRF_Auto; + if( p->spec.eBlob>QRF_BLOB_Size ) p->spec.eBlob = QRF_Auto; +qrf_reinit: + switch( p->spec.eStyle ){ + case QRF_Auto: { + switch( sqlite3_stmt_isexplain(pStmt) ){ + case 0: p->spec.eStyle = QRF_STYLE_Box; break; + case 1: p->spec.eStyle = QRF_STYLE_Explain; break; + default: p->spec.eStyle = QRF_STYLE_Eqp; break; + } + goto qrf_reinit; + } + case QRF_STYLE_List: { + if( p->spec.zColumnSep==0 ) p->spec.zColumnSep = "|"; + if( p->spec.zRowSep==0 ) p->spec.zRowSep = "\n"; + break; + } + case QRF_STYLE_JObject: + case QRF_STYLE_Json: { + p->spec.eText = QRF_TEXT_Json; + p->spec.zNull = "null"; + break; + } + case QRF_STYLE_Html: { + p->spec.eText = QRF_TEXT_Html; + p->spec.zNull = "null"; + break; + } + case QRF_STYLE_Insert: { + p->spec.eText = QRF_TEXT_Sql; + p->spec.zNull = "NULL"; + if( p->spec.zTableName==0 || p->spec.zTableName[0]==0 ){ + p->spec.zTableName = "tab"; + } + p->u.nIns = 0; + break; + } + case QRF_STYLE_Line: { + if( p->spec.zColumnSep==0 ){ + p->spec.zColumnSep = ": "; + } + break; + } + case QRF_STYLE_Csv: { + p->spec.eStyle = QRF_STYLE_List; + p->spec.eText = QRF_TEXT_Csv; + p->spec.zColumnSep = ","; + p->spec.zRowSep = "\r\n"; + p->spec.zNull = ""; + break; + } + case QRF_STYLE_Quote: { + p->spec.eText = QRF_TEXT_Sql; + p->spec.zNull = "NULL"; + p->spec.zColumnSep = ","; + p->spec.zRowSep = "\n"; + break; + } + case QRF_STYLE_Eqp: { + int expMode = sqlite3_stmt_isexplain(p->pStmt); + if( expMode!=2 ){ + sqlite3_stmt_explain(p->pStmt, 2); + p->expMode = expMode+1; + } + break; + } + case QRF_STYLE_Explain: { + int expMode = sqlite3_stmt_isexplain(p->pStmt); + if( expMode!=1 ){ + sqlite3_stmt_explain(p->pStmt, 1); + p->expMode = expMode+1; + } + break; + } + } + if( p->spec.eEsc==QRF_Auto ){ + p->spec.eEsc = QRF_ESC_Ascii; + } + if( p->spec.eText==QRF_Auto ){ + p->spec.eText = QRF_TEXT_Plain; + } + if( p->spec.eTitle==QRF_Auto ){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + case QRF_STYLE_Column: + case QRF_STYLE_Table: + p->spec.eTitle = QRF_TEXT_Plain; + break; + default: + p->spec.eTitle = p->spec.eText; + break; + } + } + if( p->spec.eBlob==QRF_Auto ){ + switch( p->spec.eText ){ + case QRF_TEXT_Sql: p->spec.eBlob = QRF_BLOB_Sql; break; + case QRF_TEXT_Csv: p->spec.eBlob = QRF_BLOB_Tcl; break; + case QRF_TEXT_Tcl: p->spec.eBlob = QRF_BLOB_Tcl; break; + case QRF_TEXT_Json: p->spec.eBlob = QRF_BLOB_Json; break; + default: p->spec.eBlob = QRF_BLOB_Text; break; + } + } + if( p->spec.bTitles==QRF_Auto ){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + case QRF_STYLE_Csv: + case QRF_STYLE_Column: + case QRF_STYLE_Table: + case QRF_STYLE_Markdown: + p->spec.bTitles = QRF_Yes; + break; + default: + p->spec.bTitles = QRF_No; + break; + } + } + if( p->spec.bWordWrap==QRF_Auto ){ + p->spec.bWordWrap = QRF_Yes; + } + if( p->spec.bTextJsonb==QRF_Auto ){ + p->spec.bTextJsonb = QRF_No; + } + if( p->spec.zColumnSep==0 ) p->spec.zColumnSep = ","; + if( p->spec.zRowSep==0 ) p->spec.zRowSep = "\n"; +} /* -** A single step of the Keccak mixing function for a 1600-bit state +** Finish rendering the results */ -static void KeccakF1600Step(SHA3Context *p){ - int i; - u64 b0, b1, b2, b3, b4; - u64 c0, c1, c2, c3, c4; - u64 d0, d1, d2, d3, d4; - static const u64 RC[] = { - 0x0000000000000001ULL, 0x0000000000008082ULL, - 0x800000000000808aULL, 0x8000000080008000ULL, - 0x000000000000808bULL, 0x0000000080000001ULL, - 0x8000000080008081ULL, 0x8000000000008009ULL, - 0x000000000000008aULL, 0x0000000000000088ULL, - 0x0000000080008009ULL, 0x000000008000000aULL, - 0x000000008000808bULL, 0x800000000000008bULL, - 0x8000000000008089ULL, 0x8000000000008003ULL, - 0x8000000000008002ULL, 0x8000000000000080ULL, - 0x000000000000800aULL, 0x800000008000000aULL, - 0x8000000080008081ULL, 0x8000000000008080ULL, - 0x0000000080000001ULL, 0x8000000080008008ULL - }; -# define a00 (p->u.s[0]) -# define a01 (p->u.s[1]) -# define a02 (p->u.s[2]) -# define a03 (p->u.s[3]) -# define a04 (p->u.s[4]) -# define a10 (p->u.s[5]) -# define a11 (p->u.s[6]) -# define a12 (p->u.s[7]) -# define a13 (p->u.s[8]) -# define a14 (p->u.s[9]) -# define a20 (p->u.s[10]) -# define a21 (p->u.s[11]) -# define a22 (p->u.s[12]) -# define a23 (p->u.s[13]) -# define a24 (p->u.s[14]) -# define a30 (p->u.s[15]) -# define a31 (p->u.s[16]) -# define a32 (p->u.s[17]) -# define a33 (p->u.s[18]) -# define a34 (p->u.s[19]) -# define a40 (p->u.s[20]) -# define a41 (p->u.s[21]) -# define a42 (p->u.s[22]) -# define a43 (p->u.s[23]) -# define a44 (p->u.s[24]) -# define ROL64(a,x) ((a<>(64-x))) +static void qrfFinalize(Qrf *p){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Count: { + sqlite3_str_appendf(p->pOut, "%lld\n", p->nRow); + break; + } + case QRF_STYLE_Json: { + if( p->nRow>0 ){ + sqlite3_str_append(p->pOut, "}]\n", 3); + } + break; + } + case QRF_STYLE_JObject: { + if( p->nRow>0 ){ + sqlite3_str_append(p->pOut, "}\n", 2); + } + break; + } + case QRF_STYLE_Insert: { + if( p->u.nIns ){ + sqlite3_str_append(p->pOut, ";\n", 2); + } + break; + } + case QRF_STYLE_Line: { + if( p->u.sLine.azCol ){ + int i; + for(i=0; inCol; i++) sqlite3_free(p->u.sLine.azCol[i]); + sqlite3_free(p->u.sLine.azCol); + } + break; + } + case QRF_STYLE_Stats: + case QRF_STYLE_StatsEst: { + i64 nCycle = 0; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + sqlite3_stmt_scanstatus_v2(p->pStmt, -1, SQLITE_SCANSTAT_NCYCLE, + SQLITE_SCANSTAT_COMPLEX, (void*)&nCycle); +#endif + qrfEqpRender(p, nCycle); + break; + } + case QRF_STYLE_Eqp: { + qrfEqpRender(p, 0); + break; + } + } + qrfWrite(p); + qrfStrErr(p, p->pOut); + if( p->spec.pzOutput ){ + if( p->spec.pzOutput[0] ){ + sqlite3_int64 n, sz; + char *zCombined; + sz = strlen(p->spec.pzOutput[0]); + n = sqlite3_str_length(p->pOut); + zCombined = sqlite3_realloc64(p->spec.pzOutput[0], sz+n+1); + if( zCombined==0 ){ + sqlite3_free(p->spec.pzOutput[0]); + p->spec.pzOutput[0] = 0; + qrfOom(p); + }else{ + p->spec.pzOutput[0] = zCombined; + memcpy(zCombined+sz, sqlite3_str_value(p->pOut), n+1); + } + sqlite3_free(sqlite3_str_finish(p->pOut)); + }else{ + p->spec.pzOutput[0] = sqlite3_str_finish(p->pOut); + } + }else if( p->pOut ){ + sqlite3_free(sqlite3_str_finish(p->pOut)); + } + if( p->expMode>0 ){ + sqlite3_stmt_explain(p->pStmt, p->expMode-1); + } + if( p->actualWidth ){ + sqlite3_free(p->actualWidth); + } + if( p->pJTrans ){ + sqlite3 *db = sqlite3_db_handle(p->pJTrans); + sqlite3_finalize(p->pJTrans); + sqlite3_close(db); + } +} - for(i=0; i<24; i+=4){ - c0 = a00^a10^a20^a30^a40; - c1 = a01^a11^a21^a31^a41; - c2 = a02^a12^a22^a32^a42; - c3 = a03^a13^a23^a33^a43; - c4 = a04^a14^a24^a34^a44; - d0 = c4^ROL64(c1, 1); - d1 = c0^ROL64(c2, 1); - d2 = c1^ROL64(c3, 1); - d3 = c2^ROL64(c4, 1); - d4 = c3^ROL64(c0, 1); +/* +** Run the prepared statement pStmt and format the results according +** to the specification provided in pSpec. Return an error code. +** If pzErr is not NULL and if an error occurs, write an error message +** into *pzErr. +*/ +int sqlite3_format_query_result( + sqlite3_stmt *pStmt, /* Statement to evaluate */ + const sqlite3_qrf_spec *pSpec, /* Format specification */ + char **pzErr /* Write error message here */ +){ + Qrf qrf; /* The new Qrf being created */ + + if( pStmt==0 ) return SQLITE_OK; /* No-op */ + if( pSpec==0 ) return SQLITE_MISUSE; + qrfInitialize(&qrf, pStmt, pSpec, pzErr); + switch( qrf.spec.eStyle ){ + case QRF_STYLE_Box: + case QRF_STYLE_Column: + case QRF_STYLE_Markdown: + case QRF_STYLE_Table: { + /* Columnar modes require that the entire query be evaluated and the + ** results stored in memory, so that we can compute column widths */ + qrfColumnar(&qrf); + break; + } + case QRF_STYLE_Explain: { + qrfExplain(&qrf); + break; + } + case QRF_STYLE_StatsVm: { + qrfScanStatusVm(&qrf); + break; + } + case QRF_STYLE_Stats: + case QRF_STYLE_StatsEst: { + qrfEqpStats(&qrf); + break; + } + default: { + /* Non-columnar modes where the output can occur after each row + ** of result is received */ + while( qrf.iErr==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + qrfOneSimpleRow(&qrf); + } + break; + } + } + qrfResetStmt(&qrf); + qrfFinalize(&qrf); + return qrf.iErr; +} - b0 = (a00^d0); - b1 = ROL64((a11^d1), 44); - b2 = ROL64((a22^d2), 43); - b3 = ROL64((a33^d3), 21); - b4 = ROL64((a44^d4), 14); - a00 = b0 ^((~b1)& b2 ); - a00 ^= RC[i]; - a11 = b1 ^((~b2)& b3 ); - a22 = b2 ^((~b3)& b4 ); - a33 = b3 ^((~b4)& b0 ); - a44 = b4 ^((~b0)& b1 ); +/************************* End ext/qrf/qrf.c ********************/ - b2 = ROL64((a20^d0), 3); - b3 = ROL64((a31^d1), 45); - b4 = ROL64((a42^d2), 61); - b0 = ROL64((a03^d3), 28); - b1 = ROL64((a14^d4), 20); - a20 = b0 ^((~b1)& b2 ); - a31 = b1 ^((~b2)& b3 ); - a42 = b2 ^((~b3)& b4 ); - a03 = b3 ^((~b4)& b0 ); - a14 = b4 ^((~b0)& b1 ); +/* Use console I/O package as a direct INCLUDE. */ +#define SQLITE_INTERNAL_LINKAGE static - b4 = ROL64((a40^d0), 18); - b0 = ROL64((a01^d1), 1); - b1 = ROL64((a12^d2), 6); - b2 = ROL64((a23^d3), 25); - b3 = ROL64((a34^d4), 8); - a40 = b0 ^((~b1)& b2 ); - a01 = b1 ^((~b2)& b3 ); - a12 = b2 ^((~b3)& b4 ); - a23 = b3 ^((~b4)& b0 ); - a34 = b4 ^((~b0)& b1 ); +#ifdef SQLITE_SHELL_FIDDLE +/* Deselect most features from the console I/O package for Fiddle. */ +# define SQLITE_CIO_NO_REDIRECT +# define SQLITE_CIO_NO_CLASSIFY +# define SQLITE_CIO_NO_TRANSLATE +# define SQLITE_CIO_NO_SETMODE +# define SQLITE_CIO_NO_FLUSH +#endif - b1 = ROL64((a10^d0), 36); - b2 = ROL64((a21^d1), 10); - b3 = ROL64((a32^d2), 15); - b4 = ROL64((a43^d3), 56); - b0 = ROL64((a04^d4), 27); - a10 = b0 ^((~b1)& b2 ); - a21 = b1 ^((~b2)& b3 ); - a32 = b2 ^((~b3)& b4 ); - a43 = b3 ^((~b4)& b0 ); - a04 = b4 ^((~b0)& b1 ); - - b3 = ROL64((a30^d0), 41); - b4 = ROL64((a41^d1), 2); - b0 = ROL64((a02^d2), 62); - b1 = ROL64((a13^d3), 55); - b2 = ROL64((a24^d4), 39); - a30 = b0 ^((~b1)& b2 ); - a41 = b1 ^((~b2)& b3 ); - a02 = b2 ^((~b3)& b4 ); - a13 = b3 ^((~b4)& b0 ); - a24 = b4 ^((~b0)& b1 ); +/* +** The source code for several run-time loadable extensions is inserted +** below by the ../tool/mkshellc.tcl script. Before processing that included +** code, we need to override some macros to make the included program code +** work here in the middle of this regular program. +*/ +#define SQLITE_EXTENSION_INIT1 +#define SQLITE_EXTENSION_INIT2(X) (void)(X) - c0 = a00^a20^a40^a10^a30; - c1 = a11^a31^a01^a21^a41; - c2 = a22^a42^a12^a32^a02; - c3 = a33^a03^a23^a43^a13; - c4 = a44^a14^a34^a04^a24; - d0 = c4^ROL64(c1, 1); - d1 = c0^ROL64(c2, 1); - d2 = c1^ROL64(c3, 1); - d3 = c2^ROL64(c4, 1); - d4 = c3^ROL64(c0, 1); +/************************* Begin ext/misc/windirent.h ******************/ +/* +** 2025-06-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** An implementation of opendir(), readdir(), and closedir() for Windows, +** based on the FindFirstFile(), FindNextFile(), and FindClose() APIs +** of Win32. +** +** #include this file inside any C-code module that needs to use +** opendir()/readdir()/closedir(). This file is a no-op on non-Windows +** machines. On Windows, static functions are defined that implement +** those standard interfaces. +*/ +#if defined(_WIN32) && defined(_MSC_VER) && !defined(SQLITE_WINDIRENT_H) +#define SQLITE_WINDIRENT_H - b0 = (a00^d0); - b1 = ROL64((a31^d1), 44); - b2 = ROL64((a12^d2), 43); - b3 = ROL64((a43^d3), 21); - b4 = ROL64((a24^d4), 14); - a00 = b0 ^((~b1)& b2 ); - a00 ^= RC[i+1]; - a31 = b1 ^((~b2)& b3 ); - a12 = b2 ^((~b3)& b4 ); - a43 = b3 ^((~b4)& b0 ); - a24 = b4 ^((~b0)& b1 ); +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef FILENAME_MAX +# define FILENAME_MAX (260) +#endif +#ifndef S_ISREG +#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) +#endif +#ifndef S_ISDIR +#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) +#endif +#ifndef S_ISLNK +#define S_ISLNK(m) (0) +#endif +typedef unsigned short mode_t; - b2 = ROL64((a40^d0), 3); - b3 = ROL64((a21^d1), 45); - b4 = ROL64((a02^d2), 61); - b0 = ROL64((a33^d3), 28); - b1 = ROL64((a14^d4), 20); - a40 = b0 ^((~b1)& b2 ); - a21 = b1 ^((~b2)& b3 ); - a02 = b2 ^((~b3)& b4 ); - a33 = b3 ^((~b4)& b0 ); - a14 = b4 ^((~b0)& b1 ); +/* The dirent object for Windows is abbreviated. The only field really +** usable by applications is d_name[]. +*/ +struct dirent { + int d_ino; /* Inode number (synthesized) */ + unsigned d_attributes; /* File attributes */ + char d_name[FILENAME_MAX]; /* Null-terminated filename */ +}; - b4 = ROL64((a30^d0), 18); - b0 = ROL64((a11^d1), 1); - b1 = ROL64((a42^d2), 6); - b2 = ROL64((a23^d3), 25); - b3 = ROL64((a04^d4), 8); - a30 = b0 ^((~b1)& b2 ); - a11 = b1 ^((~b2)& b3 ); - a42 = b2 ^((~b3)& b4 ); - a23 = b3 ^((~b4)& b0 ); - a04 = b4 ^((~b0)& b1 ); +/* The internals of DIR are opaque according to standards. So it +** does not matter what we put here. */ +typedef struct DIR DIR; +struct DIR { + intptr_t d_handle; /* Handle for findfirst()/findnext() */ + struct dirent cur; /* Current entry */ +}; - b1 = ROL64((a20^d0), 36); - b2 = ROL64((a01^d1), 10); - b3 = ROL64((a32^d2), 15); - b4 = ROL64((a13^d3), 56); - b0 = ROL64((a44^d4), 27); - a20 = b0 ^((~b1)& b2 ); - a01 = b1 ^((~b2)& b3 ); - a32 = b2 ^((~b3)& b4 ); - a13 = b3 ^((~b4)& b0 ); - a44 = b4 ^((~b0)& b1 ); +/* Ignore hidden and system files */ +#define WindowsFileToIgnore(a) \ + ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) - b3 = ROL64((a10^d0), 41); - b4 = ROL64((a41^d1), 2); - b0 = ROL64((a22^d2), 62); - b1 = ROL64((a03^d3), 55); - b2 = ROL64((a34^d4), 39); - a10 = b0 ^((~b1)& b2 ); - a41 = b1 ^((~b2)& b3 ); - a22 = b2 ^((~b3)& b4 ); - a03 = b3 ^((~b4)& b0 ); - a34 = b4 ^((~b0)& b1 ); +/* +** Close a previously opened directory +*/ +static int closedir(DIR *pDir){ + int rc = 0; + if( pDir==0 ){ + return EINVAL; + } + if( pDir->d_handle!=0 && pDir->d_handle!=(-1) ){ + rc = _findclose(pDir->d_handle); + } + sqlite3_free(pDir); + return rc; +} - c0 = a00^a40^a30^a20^a10; - c1 = a31^a21^a11^a01^a41; - c2 = a12^a02^a42^a32^a22; - c3 = a43^a33^a23^a13^a03; - c4 = a24^a14^a04^a44^a34; - d0 = c4^ROL64(c1, 1); - d1 = c0^ROL64(c2, 1); - d2 = c1^ROL64(c3, 1); - d3 = c2^ROL64(c4, 1); - d4 = c3^ROL64(c0, 1); +/* +** Open a new directory. The directory name should be UTF-8 encoded. +** appropriate translations happen automatically. +*/ +static DIR *opendir(const char *zDirName){ + DIR *pDir; + wchar_t *b1; + sqlite3_int64 sz; + struct _wfinddata_t data; - b0 = (a00^d0); - b1 = ROL64((a21^d1), 44); - b2 = ROL64((a42^d2), 43); - b3 = ROL64((a13^d3), 21); - b4 = ROL64((a34^d4), 14); - a00 = b0 ^((~b1)& b2 ); - a00 ^= RC[i+2]; - a21 = b1 ^((~b2)& b3 ); - a42 = b2 ^((~b3)& b4 ); - a13 = b3 ^((~b4)& b0 ); - a34 = b4 ^((~b0)& b1 ); + pDir = sqlite3_malloc64( sizeof(DIR) ); + if( pDir==0 ) return 0; + memset(pDir, 0, sizeof(DIR)); + memset(&data, 0, sizeof(data)); + sz = strlen(zDirName); + b1 = sqlite3_malloc64( (sz+3)*sizeof(b1[0]) ); + if( b1==0 ){ + closedir(pDir); + return NULL; + } + sz = MultiByteToWideChar(CP_UTF8, 0, zDirName, sz, b1, sz); + b1[sz++] = '\\'; + b1[sz++] = '*'; + b1[sz] = 0; + if( sz+1>sizeof(data.name)/sizeof(data.name[0]) ){ + closedir(pDir); + sqlite3_free(b1); + return NULL; + } + memcpy(data.name, b1, (sz+1)*sizeof(b1[0])); + sqlite3_free(b1); + pDir->d_handle = _wfindfirst(data.name, &data); + if( pDir->d_handle<0 ){ + closedir(pDir); + return NULL; + } + while( WindowsFileToIgnore(data) ){ + memset(&data, 0, sizeof(data)); + if( _wfindnext(pDir->d_handle, &data)==-1 ){ + closedir(pDir); + return NULL; + } + } + pDir->cur.d_ino = 0; + pDir->cur.d_attributes = data.attrib; + WideCharToMultiByte(CP_UTF8, 0, data.name, -1, + pDir->cur.d_name, FILENAME_MAX, 0, 0); + return pDir; +} - b2 = ROL64((a30^d0), 3); - b3 = ROL64((a01^d1), 45); - b4 = ROL64((a22^d2), 61); - b0 = ROL64((a43^d3), 28); - b1 = ROL64((a14^d4), 20); - a30 = b0 ^((~b1)& b2 ); - a01 = b1 ^((~b2)& b3 ); - a22 = b2 ^((~b3)& b4 ); - a43 = b3 ^((~b4)& b0 ); - a14 = b4 ^((~b0)& b1 ); +/* +** Read the next entry from a directory. +** +** The returned struct-dirent object is managed by DIR. It is only +** valid until the next readdir() or closedir() call. Only the +** d_name[] field is meaningful. The d_name[] value has been +** translated into UTF8. +*/ +static struct dirent *readdir(DIR *pDir){ + struct _wfinddata_t data; + if( pDir==0 ) return 0; + if( (pDir->cur.d_ino++)==0 ){ + return &pDir->cur; + } + do{ + memset(&data, 0, sizeof(data)); + if( _wfindnext(pDir->d_handle, &data)==-1 ){ + return NULL; + } + }while( WindowsFileToIgnore(data) ); + pDir->cur.d_attributes = data.attrib; + WideCharToMultiByte(CP_UTF8, 0, data.name, -1, + pDir->cur.d_name, FILENAME_MAX, 0, 0); + return &pDir->cur; +} - b4 = ROL64((a10^d0), 18); - b0 = ROL64((a31^d1), 1); - b1 = ROL64((a02^d2), 6); - b2 = ROL64((a23^d3), 25); - b3 = ROL64((a44^d4), 8); - a10 = b0 ^((~b1)& b2 ); - a31 = b1 ^((~b2)& b3 ); - a02 = b2 ^((~b3)& b4 ); - a23 = b3 ^((~b4)& b0 ); - a44 = b4 ^((~b0)& b1 ); +#endif /* defined(_WIN32) && defined(_MSC_VER) */ - b1 = ROL64((a40^d0), 36); - b2 = ROL64((a11^d1), 10); - b3 = ROL64((a32^d2), 15); - b4 = ROL64((a03^d3), 56); - b0 = ROL64((a24^d4), 27); - a40 = b0 ^((~b1)& b2 ); - a11 = b1 ^((~b2)& b3 ); - a32 = b2 ^((~b3)& b4 ); - a03 = b3 ^((~b4)& b0 ); - a24 = b4 ^((~b0)& b1 ); +/************************* End ext/misc/windirent.h ********************/ +/************************* Begin ext/misc/memtrace.c ******************/ +/* +** 2019-01-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an extension that uses the SQLITE_CONFIG_MALLOC +** mechanism to add a tracing layer on top of SQLite. If this extension +** is registered prior to sqlite3_initialize(), it will cause all memory +** allocation activities to be logged on standard output, or to some other +** FILE specified by the initializer. +** +** This file needs to be compiled into the application that uses it. +** +** This extension is used to implement the --memtrace option of the +** command-line shell. +*/ +#include +#include +#include - b3 = ROL64((a20^d0), 41); - b4 = ROL64((a41^d1), 2); - b0 = ROL64((a12^d2), 62); - b1 = ROL64((a33^d3), 55); - b2 = ROL64((a04^d4), 39); - a20 = b0 ^((~b1)& b2 ); - a41 = b1 ^((~b2)& b3 ); - a12 = b2 ^((~b3)& b4 ); - a33 = b3 ^((~b4)& b0 ); - a04 = b4 ^((~b0)& b1 ); +/* The original memory allocation routines */ +static sqlite3_mem_methods memtraceBase; +static FILE *memtraceOut; - c0 = a00^a30^a10^a40^a20; - c1 = a21^a01^a31^a11^a41; - c2 = a42^a22^a02^a32^a12; - c3 = a13^a43^a23^a03^a33; - c4 = a34^a14^a44^a24^a04; - d0 = c4^ROL64(c1, 1); - d1 = c0^ROL64(c2, 1); - d2 = c1^ROL64(c3, 1); - d3 = c2^ROL64(c4, 1); - d4 = c3^ROL64(c0, 1); - - b0 = (a00^d0); - b1 = ROL64((a01^d1), 44); - b2 = ROL64((a02^d2), 43); - b3 = ROL64((a03^d3), 21); - b4 = ROL64((a04^d4), 14); - a00 = b0 ^((~b1)& b2 ); - a00 ^= RC[i+3]; - a01 = b1 ^((~b2)& b3 ); - a02 = b2 ^((~b3)& b4 ); - a03 = b3 ^((~b4)& b0 ); - a04 = b4 ^((~b0)& b1 ); - - b2 = ROL64((a10^d0), 3); - b3 = ROL64((a11^d1), 45); - b4 = ROL64((a12^d2), 61); - b0 = ROL64((a13^d3), 28); - b1 = ROL64((a14^d4), 20); - a10 = b0 ^((~b1)& b2 ); - a11 = b1 ^((~b2)& b3 ); - a12 = b2 ^((~b3)& b4 ); - a13 = b3 ^((~b4)& b0 ); - a14 = b4 ^((~b0)& b1 ); - - b4 = ROL64((a20^d0), 18); - b0 = ROL64((a21^d1), 1); - b1 = ROL64((a22^d2), 6); - b2 = ROL64((a23^d3), 25); - b3 = ROL64((a24^d4), 8); - a20 = b0 ^((~b1)& b2 ); - a21 = b1 ^((~b2)& b3 ); - a22 = b2 ^((~b3)& b4 ); - a23 = b3 ^((~b4)& b0 ); - a24 = b4 ^((~b0)& b1 ); +/* Methods that trace memory allocations */ +static void *memtraceMalloc(int n){ + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: allocate %d bytes\n", + memtraceBase.xRoundup(n)); + } + return memtraceBase.xMalloc(n); +} +static void memtraceFree(void *p){ + if( p==0 ) return; + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: free %d bytes\n", memtraceBase.xSize(p)); + } + memtraceBase.xFree(p); +} +static void *memtraceRealloc(void *p, int n){ + if( p==0 ) return memtraceMalloc(n); + if( n==0 ){ + memtraceFree(p); + return 0; + } + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: resize %d -> %d bytes\n", + memtraceBase.xSize(p), memtraceBase.xRoundup(n)); + } + return memtraceBase.xRealloc(p, n); +} +static int memtraceSize(void *p){ + return memtraceBase.xSize(p); +} +static int memtraceRoundup(int n){ + return memtraceBase.xRoundup(n); +} +static int memtraceInit(void *p){ + return memtraceBase.xInit(p); +} +static void memtraceShutdown(void *p){ + memtraceBase.xShutdown(p); +} - b1 = ROL64((a30^d0), 36); - b2 = ROL64((a31^d1), 10); - b3 = ROL64((a32^d2), 15); - b4 = ROL64((a33^d3), 56); - b0 = ROL64((a34^d4), 27); - a30 = b0 ^((~b1)& b2 ); - a31 = b1 ^((~b2)& b3 ); - a32 = b2 ^((~b3)& b4 ); - a33 = b3 ^((~b4)& b0 ); - a34 = b4 ^((~b0)& b1 ); +/* The substitute memory allocator */ +static sqlite3_mem_methods ersaztMethods = { + memtraceMalloc, + memtraceFree, + memtraceRealloc, + memtraceSize, + memtraceRoundup, + memtraceInit, + memtraceShutdown, + 0 +}; - b3 = ROL64((a40^d0), 41); - b4 = ROL64((a41^d1), 2); - b0 = ROL64((a42^d2), 62); - b1 = ROL64((a43^d3), 55); - b2 = ROL64((a44^d4), 39); - a40 = b0 ^((~b1)& b2 ); - a41 = b1 ^((~b2)& b3 ); - a42 = b2 ^((~b3)& b4 ); - a43 = b3 ^((~b4)& b0 ); - a44 = b4 ^((~b0)& b1 ); +/* Begin tracing memory allocations to out. */ +int sqlite3MemTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETMALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &ersaztMethods); + } } + memtraceOut = out; + return rc; } -/* -** Initialize a new hash. iSize determines the size of the hash -** in bits and should be one of 224, 256, 384, or 512. Or iSize -** can be zero to use the default hash size of 256 bits. -*/ -static void SHA3Init(SHA3Context *p, int iSize){ - memset(p, 0, sizeof(*p)); - if( iSize>=128 && iSize<=512 ){ - p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; - }else{ - p->nRate = (1600 - 2*256)/8; - } -#if SHA3_BYTEORDER==1234 - /* Known to be little-endian at compile-time. No-op */ -#elif SHA3_BYTEORDER==4321 - p->ixMask = 7; /* Big-endian */ -#else - { - static unsigned int one = 1; - if( 1==*(unsigned char*)&one ){ - /* Little endian. No byte swapping. */ - p->ixMask = 0; - }else{ - /* Big endian. Byte swap. */ - p->ixMask = 7; +/* Deactivate memory tracing */ +int sqlite3MemTraceDeactivate(void){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + memset(&memtraceBase, 0, sizeof(memtraceBase)); } } -#endif + memtraceOut = 0; + return rc; } +/************************* End ext/misc/memtrace.c ********************/ +/************************* Begin ext/misc/pcachetrace.c ******************/ /* -** Make consecutive calls to the SHA3Update function to add new content -** to the hash +** 2023-06-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an extension that uses the SQLITE_CONFIG_PCACHE2 +** mechanism to add a tracing layer on top of pluggable page cache of +** SQLite. If this extension is registered prior to sqlite3_initialize(), +** it will cause all page cache activities to be logged on standard output, +** or to some other FILE specified by the initializer. +** +** This file needs to be compiled into the application that uses it. +** +** This extension is used to implement the --pcachetrace option of the +** command-line shell. */ -static void SHA3Update( - SHA3Context *p, - const unsigned char *aData, - unsigned int nData +#include +#include +#include + +/* The original page cache routines */ +static sqlite3_pcache_methods2 pcacheBase; +static FILE *pcachetraceOut; + +/* Methods that trace pcache activity */ +static int pcachetraceInit(void *pArg){ + int nRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xInit(%p)\n", pArg); + } + nRes = pcacheBase.xInit(pArg); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xInit(%p) -> %d\n", pArg, nRes); + } + return nRes; +} +static void pcachetraceShutdown(void *pArg){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xShutdown(%p)\n", pArg); + } + pcacheBase.xShutdown(pArg); +} +static sqlite3_pcache *pcachetraceCreate(int szPage, int szExtra, int bPurge){ + sqlite3_pcache *pRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCreate(%d,%d,%d)\n", + szPage, szExtra, bPurge); + } + pRes = pcacheBase.xCreate(szPage, szExtra, bPurge); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCreate(%d,%d,%d) -> %p\n", + szPage, szExtra, bPurge, pRes); + } + return pRes; +} +static void pcachetraceCachesize(sqlite3_pcache *p, int nCachesize){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCachesize(%p, %d)\n", p, nCachesize); + } + pcacheBase.xCachesize(p, nCachesize); +} +static int pcachetracePagecount(sqlite3_pcache *p){ + int nRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xPagecount(%p)\n", p); + } + nRes = pcacheBase.xPagecount(p); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xPagecount(%p) -> %d\n", p, nRes); + } + return nRes; +} +static sqlite3_pcache_page *pcachetraceFetch( + sqlite3_pcache *p, + unsigned key, + int crFg ){ - unsigned int i = 0; -#if SHA3_BYTEORDER==1234 - if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ - for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; - p->nLoaded += 8; - if( p->nLoaded>=p->nRate ){ - KeccakF1600Step(p); - p->nLoaded = 0; - } - } + sqlite3_pcache_page *pRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xFetch(%p,%u,%d)\n", p, key, crFg); } -#endif - for(; iu.x[p->nLoaded] ^= aData[i]; -#elif SHA3_BYTEORDER==4321 - p->u.x[p->nLoaded^0x07] ^= aData[i]; -#else - p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; -#endif - p->nLoaded++; - if( p->nLoaded==p->nRate ){ - KeccakF1600Step(p); - p->nLoaded = 0; - } + pRes = pcacheBase.xFetch(p, key, crFg); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xFetch(%p,%u,%d) -> %p\n", + p, key, crFg, pRes); + } + return pRes; +} +static void pcachetraceUnpin( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + int bDiscard +){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xUnpin(%p, %p, %d)\n", + p, pPg, bDiscard); } + pcacheBase.xUnpin(p, pPg, bDiscard); } +static void pcachetraceRekey( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + unsigned oldKey, + unsigned newKey +){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xRekey(%p, %p, %u, %u)\n", + p, pPg, oldKey, newKey); + } + pcacheBase.xRekey(p, pPg, oldKey, newKey); +} +static void pcachetraceTruncate(sqlite3_pcache *p, unsigned n){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xTruncate(%p, %u)\n", p, n); + } + pcacheBase.xTruncate(p, n); +} +static void pcachetraceDestroy(sqlite3_pcache *p){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xDestroy(%p)\n", p); + } + pcacheBase.xDestroy(p); +} +static void pcachetraceShrink(sqlite3_pcache *p){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xShrink(%p)\n", p); + } + pcacheBase.xShrink(p); +} + +/* The substitute pcache methods */ +static sqlite3_pcache_methods2 ersaztPcacheMethods = { + 0, + 0, + pcachetraceInit, + pcachetraceShutdown, + pcachetraceCreate, + pcachetraceCachesize, + pcachetracePagecount, + pcachetraceFetch, + pcachetraceUnpin, + pcachetraceRekey, + pcachetraceTruncate, + pcachetraceDestroy, + pcachetraceShrink +}; -/* -** After all content has been added, invoke SHA3Final() to compute -** the final hash. The function returns a pointer to the binary -** hash value. -*/ -static unsigned char *SHA3Final(SHA3Context *p){ - unsigned int i; - if( p->nLoaded==p->nRate-1 ){ - const unsigned char c1 = 0x86; - SHA3Update(p, &c1, 1); - }else{ - const unsigned char c2 = 0x06; - const unsigned char c3 = 0x80; - SHA3Update(p, &c2, 1); - p->nLoaded = p->nRate - 1; - SHA3Update(p, &c3, 1); +/* Begin tracing memory allocations to out. */ +int sqlite3PcacheTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( pcacheBase.xFetch==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETPCACHE2, &pcacheBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_PCACHE2, &ersaztPcacheMethods); + } } - for(i=0; inRate; i++){ - p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + pcachetraceOut = out; + return rc; +} + +/* Deactivate memory tracing */ +int sqlite3PcacheTraceDeactivate(void){ + int rc = SQLITE_OK; + if( pcacheBase.xFetch!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_PCACHE2, &pcacheBase); + if( rc==SQLITE_OK ){ + memset(&pcacheBase, 0, sizeof(pcacheBase)); + } } - return &p->u.x[p->nRate]; + pcachetraceOut = 0; + return rc; } -/* End of the hashing logic -*****************************************************************************/ +/************************* End ext/misc/pcachetrace.c ********************/ +/************************* Begin ext/misc/shathree.c ******************/ /* -** Implementation of the sha3(X,SIZE) function. -** -** Return a BLOB which is the SIZE-bit SHA3 hash of X. The default -** size is 256. If X is a BLOB, it is hashed as is. -** For all other non-NULL types of input, X is converted into a UTF-8 string -** and the string is hashed without the trailing 0x00 terminator. The hash -** of a NULL value is NULL. -*/ -static void sha3Func( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - SHA3Context cx; - int eType = sqlite3_value_type(argv[0]); - int nByte = sqlite3_value_bytes(argv[0]); - int iSize; - if( argc==1 ){ - iSize = 256; - }else{ - iSize = sqlite3_value_int(argv[1]); - if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ - sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " - "384 512", -1); - return; - } - } - if( eType==SQLITE_NULL ) return; - SHA3Init(&cx, iSize); - if( eType==SQLITE_BLOB ){ - SHA3Update(&cx, sqlite3_value_blob(argv[0]), nByte); - }else{ - SHA3Update(&cx, sqlite3_value_text(argv[0]), nByte); - } - sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); -} - -/* Compute a string using sqlite3_vsnprintf() with a maximum length -** of 50 bytes and add it to the hash. -*/ -static void hash_step_vformat( - SHA3Context *p, /* Add content to this context */ - const char *zFormat, - ... -){ - va_list ap; - int n; - char zBuf[50]; - va_start(ap, zFormat); - sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); - va_end(ap); - n = (int)strlen(zBuf); - SHA3Update(p, (unsigned char*)zBuf, n); -} - -/* -** Implementation of the sha3_query(SQL,SIZE) function. -** -** This function compiles and runs the SQL statement(s) given in the -** argument. The results are hashed using a SIZE-bit SHA3. The default -** size is 256. -** -** The format of the byte stream that is hashed is summarized as follows: -** -** S: -** R -** N -** I -** F -** B: -** T: -** -** is the original SQL text for each statement run and is -** the size of that text. The SQL text is UTF-8. A single R character -** occurs before the start of each row. N means a NULL value. -** I mean an 8-byte little-endian integer . F is a floating point -** number with an 8-byte little-endian IEEE floating point value . -** B means blobs of bytes. T means text rendered as -** bytes of UTF-8. The and values are expressed as an ASCII -** text integers. -** -** For each SQL statement in the X input, there is one S segment. Each -** S segment is followed by zero or more R segments, one for each row in the -** result set. After each R, there are one or more N, I, F, B, or T segments, -** one for each column in the result set. Segments are concatentated directly -** with no delimiters of any kind. -*/ -static void sha3QueryFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - sqlite3 *db = sqlite3_context_db_handle(context); - const char *zSql = (const char*)sqlite3_value_text(argv[0]); - sqlite3_stmt *pStmt = 0; - int nCol; /* Number of columns in the result set */ - int i; /* Loop counter */ - int rc; - int n; - const char *z; - SHA3Context cx; - int iSize; - - if( argc==1 ){ - iSize = 256; - }else{ - iSize = sqlite3_value_int(argv[1]); - if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ - sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " - "384 512", -1); - return; - } - } - if( zSql==0 ) return; - SHA3Init(&cx, iSize); - while( zSql[0] ){ - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); - if( rc ){ - char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", - zSql, sqlite3_errmsg(db)); - sqlite3_finalize(pStmt); - sqlite3_result_error(context, zMsg, -1); - sqlite3_free(zMsg); - return; - } - if( !sqlite3_stmt_readonly(pStmt) ){ - char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); - sqlite3_finalize(pStmt); - sqlite3_result_error(context, zMsg, -1); - sqlite3_free(zMsg); - return; - } - nCol = sqlite3_column_count(pStmt); - z = sqlite3_sql(pStmt); - n = (int)strlen(z); - hash_step_vformat(&cx,"S%d:",n); - SHA3Update(&cx,(unsigned char*)z,n); - - /* Compute a hash over the result of the query */ - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - SHA3Update(&cx,(const unsigned char*)"R",1); - for(i=0; i=1; j--){ - x[j] = u & 0xff; - u >>= 8; - } - x[0] = 'I'; - SHA3Update(&cx, x, 9); - break; - } - case SQLITE_FLOAT: { - sqlite3_uint64 u; - int j; - unsigned char x[9]; - double r = sqlite3_column_double(pStmt,i); - memcpy(&u, &r, 8); - for(j=8; j>=1; j--){ - x[j] = u & 0xff; - u >>= 8; - } - x[0] = 'F'; - SHA3Update(&cx,x,9); - break; - } - case SQLITE_TEXT: { - int n2 = sqlite3_column_bytes(pStmt, i); - const unsigned char *z2 = sqlite3_column_text(pStmt, i); - hash_step_vformat(&cx,"T%d:",n2); - SHA3Update(&cx, z2, n2); - break; - } - case SQLITE_BLOB: { - int n2 = sqlite3_column_bytes(pStmt, i); - const unsigned char *z2 = sqlite3_column_blob(pStmt, i); - hash_step_vformat(&cx,"B%d:",n2); - SHA3Update(&cx, z2, n2); - break; - } - } - } - } - sqlite3_finalize(pStmt); - } - sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); -} - - -#ifdef _WIN32 - -#endif -int sqlite3_shathree_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc = SQLITE_OK; - SQLITE_EXTENSION_INIT2(pApi); - (void)pzErrMsg; /* Unused parameter */ - rc = sqlite3_create_function(db, "sha3", 1, - SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, - 0, sha3Func, 0, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "sha3", 2, - SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, - 0, sha3Func, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "sha3_query", 1, - SQLITE_UTF8 | SQLITE_DIRECTONLY, - 0, sha3QueryFunc, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "sha3_query", 2, - SQLITE_UTF8 | SQLITE_DIRECTONLY, - 0, sha3QueryFunc, 0, 0); - } - return rc; -} - -/************************* End ../ext/misc/shathree.c ********************/ -/************************* Begin ../ext/misc/fileio.c ******************/ -/* -** 2014-06-13 +** 2017-03-08 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -2042,1501 +4380,10354 @@ int sqlite3_shathree_init( ** ****************************************************************************** ** -** This SQLite extension implements SQL functions readfile() and -** writefile(), and eponymous virtual type "fsdir". +** This SQLite extension implements functions that compute SHA3 hashes +** in the way described by the (U.S.) NIST FIPS 202 SHA-3 Standard. +** Three SQL functions are implemented: ** -** WRITEFILE(FILE, DATA [, MODE [, MTIME]]): +** sha3(X,SIZE) +** sha3_agg(Y,SIZE) +** sha3_query(Z,SIZE) ** -** If neither of the optional arguments is present, then this UDF -** function writes blob DATA to file FILE. If successful, the number -** of bytes written is returned. If an error occurs, NULL is returned. +** The sha3(X) function computes the SHA3 hash of the input X, or NULL if +** X is NULL. If inputs X is text, the UTF-8 rendering of that text is +** used to compute the hash. If X is a BLOB, then the binary data of the +** blob is used to compute the hash. If X is an integer or real number, +** then that number if converted into UTF-8 text and the hash is computed +** over the text. ** -** If the first option argument - MODE - is present, then it must -** be passed an integer value that corresponds to a POSIX mode -** value (file type + permissions, as returned in the stat.st_mode -** field by the stat() system call). Three types of files may -** be written/created: +** The sha3_agg(Y) function computes the SHA3 hash of all Y inputs. Since +** order is important for the hash, it is recommended that the Y expression +** by followed by an ORDER BY clause to guarantee that the inputs occur +** in the desired order. ** -** regular files: (mode & 0170000)==0100000 -** symbolic links: (mode & 0170000)==0120000 -** directories: (mode & 0170000)==0040000 +** The sha3_query(Y) function evaluates all queries in the SQL statements of Y +** and returns a hash of their results. ** -** For a directory, the DATA is ignored. For a symbolic link, it is -** interpreted as text and used as the target of the link. For a -** regular file, it is interpreted as a blob and written into the -** named file. Regardless of the type of file, its permissions are -** set to (mode & 0777) before returning. +** The SIZE argument is optional. If omitted, the SHA3-256 hash algorithm +** is used. If SIZE is included it must be one of the integers 224, 256, +** 384, or 512, to determine SHA3 hash variant that is computed. ** -** If the optional MTIME argument is present, then it is interpreted -** as an integer - the number of seconds since the unix epoch. The -** modification-time of the target file is set to this value before -** returning. +** Because the sha3_agg() and sha3_query() functions compute a hash over +** multiple values, the values are encode to use include type information. ** -** If three or more arguments are passed to this function and an -** error is encountered, an exception is raised. +** In sha3_agg(), the sequence of bytes that gets hashed for each input +** Y depends on the datatype of Y: ** -** READFILE(FILE): +** typeof(Y)='null' A single "N" is hashed. (One byte) ** -** Read and return the contents of file FILE (type blob) from disk. +** typeof(Y)='integer' The data hash is the character "I" followed +** by an 8-byte big-endian binary of the +** 64-bit signed integer. (Nine bytes total.) ** -** FSDIR: +** typeof(Y)='real' The character "F" followed by an 8-byte +** big-ending binary of the double. (Nine +** bytes total.) ** -** Used as follows: +** typeof(Y)='text' The hash is over prefix "Tnnn:" followed +** by the UTF8 encoding of the text. The "nnn" +** in the prefix is the minimum-length decimal +** representation of the octet_length of the text. +** Notice the ":" at the end of the prefix, which +** is needed to separate the prefix from the +** content in cases where the content starts +** with a digit. ** -** SELECT * FROM fsdir($path [, $dir]); +** typeof(Y)='blob' The hash is taken over prefix "Bnnn:" followed +** by the binary content of the blob. The "nnn" +** in the prefix is the minimum-length decimal +** representation of the byte-length of the blob. ** -** Parameter $path is an absolute or relative pathname. If the file that it -** refers to does not exist, it is an error. If the path refers to a regular -** file or symbolic link, it returns a single row. Or, if the path refers -** to a directory, it returns one row for the directory, and one row for each -** file within the hierarchy rooted at $path. +** According to the rules above, all of the following SELECT statements +** should return TRUE: ** -** Each row has the following columns: +** SELECT sha3(1) = sha3('1'); ** -** name: Path to file or directory (text value). -** mode: Value of stat.st_mode for directory entry (an integer). -** mtime: Value of stat.st_mtime for directory entry (an integer). -** data: For a regular file, a blob containing the file data. For a -** symlink, a text value containing the text of the link. For a -** directory, NULL. +** SELECT sha3('hello') = sha3(x'68656c6c6f'); ** -** If a non-NULL value is specified for the optional $dir parameter and -** $path is a relative path, then $path is interpreted relative to $dir. -** And the paths returned in the "name" column of the table are also -** relative to directory $dir. +** WITH a(x) AS (VALUES('xyzzy')) +** SELECT sha3_agg(x) = sha3('T5:xyzzy') FROM a; +** +** WITH a(x) AS (VALUES(x'010203')) +** SELECT sha3_agg(x) = sha3(x'42333a010203') FROM a; +** +** WITH a(x) AS (VALUES(0x123456)) +** SELECT sha3_agg(x) = sha3(x'490000000000123456') FROM a; +** +** WITH a(x) AS (VALUES(100.015625)) +** SELECT sha3_agg(x) = sha3(x'464059010000000000') FROM a; +** +** WITH a(x) AS (VALUES(NULL)) +** SELECT sha3_agg(x) = sha3('N') FROM a; +** +** +** In sha3_query(), individual column values are encoded as with +** sha3_agg(), but with the addition that a single "R" character is +** inserted at the start of each row. +** +** Note that sha3_agg() hashes rows for which Y is NULL. Add a FILTER +** clause if NULL rows should be excluded: +** +** SELECT sha3_agg(x ORDER BY rowid) FILTER(WHERE x NOT NULL) FROM t1; */ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 -#include -#include #include +#include +#include -#include -#include -#include -#if !defined(_WIN32) && !defined(WIN32) -# include -# include -# include -# include -#else -# include "windows.h" -# include -# include -/* # include "test_windirent.h" */ -# define dirent DIRENT -# ifndef chmod -# define chmod _chmod -# endif -# ifndef stat -# define stat _stat -# endif -# define mkdir(path,mode) _mkdir(path) -# define lstat(path,buf) stat(path,buf) -#endif -#include -#include - +#ifndef SQLITE_AMALGAMATION +/* typedef sqlite3_uint64 u64; */ +#endif /* SQLITE_AMALGAMATION */ -/* -** Structure of the fsdir() table-valued function +/****************************************************************************** +** The Hash Engine */ - /* 0 1 2 3 4 5 */ -#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)" -#define FSDIR_COLUMN_NAME 0 /* Name of the file */ -#define FSDIR_COLUMN_MODE 1 /* Access mode */ -#define FSDIR_COLUMN_MTIME 2 /* Last modification time */ -#define FSDIR_COLUMN_DATA 3 /* File content */ -#define FSDIR_COLUMN_PATH 4 /* Path to top of search */ -#define FSDIR_COLUMN_DIR 5 /* Path is relative to this directory */ - - /* -** Set the result stored by context ctx to a blob containing the -** contents of file zName. Or, leave the result unchanged (NULL) -** if the file does not exist or is unreadable. -** -** If the file exceeds the SQLite blob size limit, through an -** SQLITE_TOOBIG error. +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. ** -** Throw an SQLITE_IOERR if there are difficulties pulling the file -** off of disk. +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSHA3_BYTEORDER=0 is set, then byte-order is determined +** at run-time. */ -static void readFileContents(sqlite3_context *ctx, const char *zName){ - FILE *in; - sqlite3_int64 nIn; - void *pBuf; - sqlite3 *db; - int mxBlob; +#ifndef SHA3_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SHA3_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define SHA3_BYTEORDER 4321 +# else +# define SHA3_BYTEORDER 0 +# endif +#endif - in = fopen(zName, "rb"); - if( in==0 ){ - /* File does not exist or is unreadable. Leave the result set to NULL. */ - return; - } - fseek(in, 0, SEEK_END); - nIn = ftell(in); - rewind(in); - db = sqlite3_context_db_handle(ctx); - mxBlob = sqlite3_limit(db, SQLITE_LIMIT_LENGTH, -1); - if( nIn>mxBlob ){ - sqlite3_result_error_code(ctx, SQLITE_TOOBIG); - fclose(in); - return; - } - pBuf = sqlite3_malloc64( nIn ? nIn : 1 ); - if( pBuf==0 ){ - sqlite3_result_error_nomem(ctx); - fclose(in); - return; - } - if( nIn==(sqlite3_int64)fread(pBuf, 1, (size_t)nIn, in) ){ - sqlite3_result_blob64(ctx, pBuf, nIn, sqlite3_free); - }else{ - sqlite3_result_error_code(ctx, SQLITE_IOERR); - sqlite3_free(pBuf); - } - fclose(in); -} /* -** Implementation of the "readfile(X)" SQL function. The entire content -** of the file named X is read and returned as a BLOB. NULL is returned -** if the file does not exist or is unreadable. +** State structure for a SHA3 hash in progress */ -static void readfileFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *zName; - (void)(argc); /* Unused parameter */ - zName = (const char*)sqlite3_value_text(argv[0]); - if( zName==0 ) return; - readFileContents(context, zName); -} +typedef struct SHA3Context SHA3Context; +struct SHA3Context { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ + unsigned iSize; /* 224, 256, 358, or 512 */ +}; /* -** Set the error message contained in context ctx to the results of -** vprintf(zFmt, ...). +** A single step of the Keccak mixing function for a 1600-bit state */ -static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ - char *zMsg = 0; - va_list ap; - va_start(ap, zFmt); - zMsg = sqlite3_vmprintf(zFmt, ap); - sqlite3_result_error(ctx, zMsg, -1); - sqlite3_free(zMsg); - va_end(ap); -} +static void KeccakF1600Step(SHA3Context *p){ + int i; + u64 b0, b1, b2, b3, b4; + u64 c0, c1, c2, c3, c4; + u64 d0, d1, d2, d3, d4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define a00 (p->u.s[0]) +# define a01 (p->u.s[1]) +# define a02 (p->u.s[2]) +# define a03 (p->u.s[3]) +# define a04 (p->u.s[4]) +# define a10 (p->u.s[5]) +# define a11 (p->u.s[6]) +# define a12 (p->u.s[7]) +# define a13 (p->u.s[8]) +# define a14 (p->u.s[9]) +# define a20 (p->u.s[10]) +# define a21 (p->u.s[11]) +# define a22 (p->u.s[12]) +# define a23 (p->u.s[13]) +# define a24 (p->u.s[14]) +# define a30 (p->u.s[15]) +# define a31 (p->u.s[16]) +# define a32 (p->u.s[17]) +# define a33 (p->u.s[18]) +# define a34 (p->u.s[19]) +# define a40 (p->u.s[20]) +# define a41 (p->u.s[21]) +# define a42 (p->u.s[22]) +# define a43 (p->u.s[23]) +# define a44 (p->u.s[24]) +# define ROL64(a,x) ((a<>(64-x))) -#if defined(_WIN32) -/* -** This function is designed to convert a Win32 FILETIME structure into the -** number of seconds since the Unix Epoch (1970-01-01 00:00:00 UTC). -*/ -static sqlite3_uint64 fileTimeToUnixTime( - LPFILETIME pFileTime -){ - SYSTEMTIME epochSystemTime; - ULARGE_INTEGER epochIntervals; - FILETIME epochFileTime; - ULARGE_INTEGER fileIntervals; + for(i=0; i<24; i+=4){ + c0 = a00^a10^a20^a30^a40; + c1 = a01^a11^a21^a31^a41; + c2 = a02^a12^a22^a32^a42; + c3 = a03^a13^a23^a33^a43; + c4 = a04^a14^a24^a34^a44; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); - memset(&epochSystemTime, 0, sizeof(SYSTEMTIME)); - epochSystemTime.wYear = 1970; - epochSystemTime.wMonth = 1; - epochSystemTime.wDay = 1; - SystemTimeToFileTime(&epochSystemTime, &epochFileTime); - epochIntervals.LowPart = epochFileTime.dwLowDateTime; - epochIntervals.HighPart = epochFileTime.dwHighDateTime; + b0 = (a00^d0); + b1 = ROL64((a11^d1), 44); + b2 = ROL64((a22^d2), 43); + b3 = ROL64((a33^d3), 21); + b4 = ROL64((a44^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i]; + a11 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); - fileIntervals.LowPart = pFileTime->dwLowDateTime; - fileIntervals.HighPart = pFileTime->dwHighDateTime; + b2 = ROL64((a20^d0), 3); + b3 = ROL64((a31^d1), 45); + b4 = ROL64((a42^d2), 61); + b0 = ROL64((a03^d3), 28); + b1 = ROL64((a14^d4), 20); + a20 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); - return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000; -} + b4 = ROL64((a40^d0), 18); + b0 = ROL64((a01^d1), 1); + b1 = ROL64((a12^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a34^d4), 8); + a40 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a10^d0), 36); + b2 = ROL64((a21^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a43^d3), 56); + b0 = ROL64((a04^d4), 27); + a10 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a30^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a02^d2), 62); + b1 = ROL64((a13^d3), 55); + b2 = ROL64((a24^d4), 39); + a30 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + c0 = a00^a20^a40^a10^a30; + c1 = a11^a31^a01^a21^a41; + c2 = a22^a42^a12^a32^a02; + c3 = a33^a03^a23^a43^a13; + c4 = a44^a14^a34^a04^a24; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a31^d1), 44); + b2 = ROL64((a12^d2), 43); + b3 = ROL64((a43^d3), 21); + b4 = ROL64((a24^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+1]; + a31 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a40^d0), 3); + b3 = ROL64((a21^d1), 45); + b4 = ROL64((a02^d2), 61); + b0 = ROL64((a33^d3), 28); + b1 = ROL64((a14^d4), 20); + a40 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a30^d0), 18); + b0 = ROL64((a11^d1), 1); + b1 = ROL64((a42^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a04^d4), 8); + a30 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a20^d0), 36); + b2 = ROL64((a01^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a13^d3), 56); + b0 = ROL64((a44^d4), 27); + a20 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a10^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a22^d2), 62); + b1 = ROL64((a03^d3), 55); + b2 = ROL64((a34^d4), 39); + a10 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + c0 = a00^a40^a30^a20^a10; + c1 = a31^a21^a11^a01^a41; + c2 = a12^a02^a42^a32^a22; + c3 = a43^a33^a23^a13^a03; + c4 = a24^a14^a04^a44^a34; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a21^d1), 44); + b2 = ROL64((a42^d2), 43); + b3 = ROL64((a13^d3), 21); + b4 = ROL64((a34^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+2]; + a21 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a30^d0), 3); + b3 = ROL64((a01^d1), 45); + b4 = ROL64((a22^d2), 61); + b0 = ROL64((a43^d3), 28); + b1 = ROL64((a14^d4), 20); + a30 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a10^d0), 18); + b0 = ROL64((a31^d1), 1); + b1 = ROL64((a02^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a44^d4), 8); + a10 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a40^d0), 36); + b2 = ROL64((a11^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a03^d3), 56); + b0 = ROL64((a24^d4), 27); + a40 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a20^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a12^d2), 62); + b1 = ROL64((a33^d3), 55); + b2 = ROL64((a04^d4), 39); + a20 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + c0 = a00^a30^a10^a40^a20; + c1 = a21^a01^a31^a11^a41; + c2 = a42^a22^a02^a32^a12; + c3 = a13^a43^a23^a03^a33; + c4 = a34^a14^a44^a24^a04; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a01^d1), 44); + b2 = ROL64((a02^d2), 43); + b3 = ROL64((a03^d3), 21); + b4 = ROL64((a04^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+3]; + a01 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a10^d0), 3); + b3 = ROL64((a11^d1), 45); + b4 = ROL64((a12^d2), 61); + b0 = ROL64((a13^d3), 28); + b1 = ROL64((a14^d4), 20); + a10 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a20^d0), 18); + b0 = ROL64((a21^d1), 1); + b1 = ROL64((a22^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a24^d4), 8); + a20 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a30^d0), 36); + b2 = ROL64((a31^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a33^d3), 56); + b0 = ROL64((a34^d4), 27); + a30 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a40^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a42^d2), 62); + b1 = ROL64((a43^d3), 55); + b2 = ROL64((a44^d4), 39); + a40 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void SHA3Init(SHA3Context *p, int iSize){ + memset(p, 0, sizeof(*p)); + p->iSize = iSize; + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if SHA3_BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif SHA3_BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the SHA3Update function to add new content +** to the hash +*/ +static void SHA3Update( + SHA3Context *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; + if( aData==0 ) return; +#if SHA3_BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; iu.x[p->nLoaded] ^= aData[i]; +#elif SHA3_BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke SHA3Final() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *SHA3Final(SHA3Context *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + SHA3Update(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + SHA3Update(p, &c2, 1); + p->nLoaded = p->nRate - 1; + SHA3Update(p, &c3, 1); + } + for(i=0; inRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} +/* End of the hashing logic +*****************************************************************************/ + +/* +** Implementation of the sha3(X,SIZE) function. +** +** Return a BLOB which is the SIZE-bit SHA3 hash of X. The default +** size is 256. If X is a BLOB, it is hashed as is. +** For all other non-NULL types of input, X is converted into a UTF-8 string +** and the string is hashed without the trailing 0x00 terminator. The hash +** of a NULL value is NULL. +*/ +static void sha3Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA3Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + int iSize; + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( eType==SQLITE_NULL ) return; + SHA3Init(&cx, iSize); + if( eType==SQLITE_BLOB ){ + SHA3Update(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + SHA3Update(&cx, sqlite3_value_text(argv[0]), nByte); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + +/* Compute a string using sqlite3_vsnprintf() with a maximum length +** of 50 bytes and add it to the hash. +*/ +static void sha3_step_vformat( + SHA3Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + SHA3Update(p, (unsigned char*)zBuf, n); +} + +/* +** Update a SHA3Context using a single sqlite3_value. +*/ +static void sha3UpdateFromValue(SHA3Context *p, sqlite3_value *pVal){ + switch( sqlite3_value_type(pVal) ){ + case SQLITE_NULL: { + SHA3Update(p, (const unsigned char*)"N",1); + break; + } + case SQLITE_INTEGER: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + sqlite3_int64 v = sqlite3_value_int64(pVal); + memcpy(&u, &v, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + SHA3Update(p, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_value_double(pVal); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + SHA3Update(p,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_value_bytes(pVal); + const unsigned char *z2 = sqlite3_value_text(pVal); + sha3_step_vformat(p,"T%d:",n2); + SHA3Update(p, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_value_bytes(pVal); + const unsigned char *z2 = sqlite3_value_blob(pVal); + sha3_step_vformat(p,"B%d:",n2); + SHA3Update(p, z2, n2); + break; + } + } +} + +/* +** Implementation of the sha3_query(SQL,SIZE) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using a SIZE-bit SHA3. The default +** size is 256. +** +** The format of the byte stream that is hashed is summarized as follows: +** +** S: +** R +** N +** I +** F +** B: +** T: +** +** is the original SQL text for each statement run and is +** the size of that text. The SQL text is UTF-8. A single R character +** occurs before the start of each row. N means a NULL value. +** I mean an 8-byte little-endian integer . F is a floating point +** number with an 8-byte little-endian IEEE floating point value . +** B means blobs of bytes. T means text rendered as +** bytes of UTF-8. The and values are expressed as an ASCII +** text integers. +** +** For each SQL statement in the X input, there is one S segment. Each +** S segment is followed by zero or more R segments, one for each row in the +** result set. After each R, there are one or more N, I, F, B, or T segments, +** one for each column in the result set. Segments are concatentated directly +** with no delimiters of any kind. +*/ +static void sha3QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA3Context cx; + int iSize; + + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( zSql==0 ) return; + SHA3Init(&cx, iSize); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + if( z ){ + n = (int)strlen(z); + sha3_step_vformat(&cx,"S%d:",n); + SHA3Update(&cx,(unsigned char*)z,n); + } + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + SHA3Update(&cx,(const unsigned char*)"R",1); + for(i=0; inRate==0 ){ + int sz = 256; + if( argc==2 ){ + sz = sqlite3_value_int(argv[1]); + if( sz!=224 && sz!=384 && sz!=512 ){ + sz = 256; + } + } + SHA3Init(p, sz); + } + sha3UpdateFromValue(p, argv[0]); +} + + +/* +** xFinal function for sha3_agg(). +*/ +static void sha3AggFinal(sqlite3_context *context){ + SHA3Context *p; + p = (SHA3Context*)sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( p->iSize ){ + sqlite3_result_blob(context, SHA3Final(p), p->iSize/8, SQLITE_TRANSIENT); + } +} + + + +#ifdef _WIN32 + +#endif +int sqlite3_shathree_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "sha3", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, sha3Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3", 2, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, sha3Func, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_agg", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, 0, sha3AggStep, sha3AggFinal); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_agg", 2, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, 0, sha3AggStep, sha3AggFinal); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_query", 1, + SQLITE_UTF8 | SQLITE_DIRECTONLY, + 0, sha3QueryFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_query", 2, + SQLITE_UTF8 | SQLITE_DIRECTONLY, + 0, sha3QueryFunc, 0, 0); + } + return rc; +} + +/************************* End ext/misc/shathree.c ********************/ +/************************* Begin ext/misc/sha1.c ******************/ +/* +** 2017-01-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements functions that compute SHA1 hashes. +** Two SQL functions are implemented: +** +** sha1(X) +** sha1_query(Y) +** +** The sha1(X) function computes the SHA1 hash of the input X, or NULL if +** X is NULL. +** +** The sha1_query(Y) function evalutes all queries in the SQL statements of Y +** and returns a hash of their results. +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +/****************************************************************************** +** The Hash Engine +*/ +/* Context for the SHA1 hash */ +typedef struct SHA1Context SHA1Context; +struct SHA1Context { + unsigned int state[5]; + unsigned int count[2]; + unsigned char buffer[64]; +}; + +#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r)) +#define rol(x,k) SHA_ROT(x,k,32-(k)) +#define ror(x,k) SHA_ROT(x,32-(k),k) + +#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \ + |(rol(block[i],8)&0x00FF00FF)) +#define blk0be(i) block[i] +#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \ + ^block[(i+2)&15]^block[i&15],1)) + +/* + * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1 + * + * Rl0() for little-endian and Rb0() for big-endian. Endianness is + * determined at run-time. + */ +#define Rl0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define Rb0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R1(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R2(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2); +#define R3(v,w,x,y,z,i) \ + z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2); +#define R4(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2); + +/* + * Hash a single 512-bit block. This is the core of the algorithm. + */ +static void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){ + unsigned int qq[5]; /* a, b, c, d, e; */ + static int one = 1; + unsigned int block[16]; + memcpy(block, buffer, 64); + memcpy(qq,state,5*sizeof(unsigned int)); + +#define a qq[0] +#define b qq[1] +#define c qq[2] +#define d qq[3] +#define e qq[4] + + /* Copy p->state[] to working vars */ + /* + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + */ + + /* 4 rounds of 20 operations each. Loop unrolled. */ + if( 1 == *(unsigned char*)&one ){ + Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3); + Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7); + Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11); + Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15); + }else{ + Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3); + Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7); + Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11); + Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15); + } + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + +#undef a +#undef b +#undef c +#undef d +#undef e +} + + +/* Initialize a SHA1 context */ +static void hash_init(SHA1Context *p){ + /* SHA1 initialization constants */ + p->state[0] = 0x67452301; + p->state[1] = 0xEFCDAB89; + p->state[2] = 0x98BADCFE; + p->state[3] = 0x10325476; + p->state[4] = 0xC3D2E1F0; + p->count[0] = p->count[1] = 0; +} + +/* Add new content to the SHA1 hash */ +static void hash_step( + SHA1Context *p, /* Add content to this context */ + const unsigned char *data, /* Data to be added */ + unsigned int len /* Number of bytes in data */ +){ + unsigned int i, j; + + j = p->count[0]; + if( (p->count[0] += len << 3) < j ){ + p->count[1] += (len>>29)+1; + } + j = (j >> 3) & 63; + if( (j + len) > 63 ){ + (void)memcpy(&p->buffer[j], data, (i = 64-j)); + SHA1Transform(p->state, p->buffer); + for(; i + 63 < len; i += 64){ + SHA1Transform(p->state, &data[i]); + } + j = 0; + }else{ + i = 0; + } + (void)memcpy(&p->buffer[j], &data[i], len - i); +} + +/* Compute a string using sqlite3_vsnprintf() and hash it */ +static void hash_step_vformat( + SHA1Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + hash_step(p, (unsigned char*)zBuf, n); +} + + +/* Add padding and compute the message digest. Render the +** message digest as lower-case hexadecimal and put it into +** zOut[]. zOut[] must be at least 41 bytes long. */ +static void hash_finish( + SHA1Context *p, /* The SHA1 context to finish and render */ + char *zOut, /* Store hex or binary hash here */ + int bAsBinary /* 1 for binary hash, 0 for hex hash */ +){ + unsigned int i; + unsigned char finalcount[8]; + unsigned char digest[20]; + static const char zEncode[] = "0123456789abcdef"; + + for (i = 0; i < 8; i++){ + finalcount[i] = (unsigned char)((p->count[(i >= 4 ? 0 : 1)] + >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } + hash_step(p, (const unsigned char *)"\200", 1); + while ((p->count[0] & 504) != 448){ + hash_step(p, (const unsigned char *)"\0", 1); + } + hash_step(p, finalcount, 8); /* Should cause a SHA1Transform() */ + for (i = 0; i < 20; i++){ + digest[i] = (unsigned char)((p->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } + if( bAsBinary ){ + memcpy(zOut, digest, 20); + }else{ + for(i=0; i<20; i++){ + zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; + zOut[i*2+1] = zEncode[digest[i] & 0xf]; + } + zOut[i*2]= 0; + } +} +/* End of the hashing logic +*****************************************************************************/ + +/* +** Two SQL functions: sha1(X) and sha1b(X). +** +** sha1(X) returns a lower-case hexadecimal rendering of the SHA1 hash +** of the argument X. If X is a BLOB, it is hashed as is. For all other +** types of input, X is converted into a UTF-8 string and the string +** is hashed without the trailing 0x00 terminator. The hash of a NULL +** value is NULL. +** +** sha1b(X) is the same except that it returns a 20-byte BLOB containing +** the binary hash instead of a hexadecimal string. +*/ +static void sha1Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA1Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + const unsigned char *pData; + char zOut[44]; + + assert( argc==1 ); + if( eType==SQLITE_NULL ) return; + hash_init(&cx); + if( eType==SQLITE_BLOB ){ + pData = (const unsigned char*)sqlite3_value_blob(argv[0]); + }else{ + pData = (const unsigned char*)sqlite3_value_text(argv[0]); + } + if( pData==0 ) return; + hash_step(&cx, pData, nByte); + if( sqlite3_user_data(context)!=0 ){ + /* sha1b() - binary result */ + hash_finish(&cx, zOut, 1); + sqlite3_result_blob(context, zOut, 20, SQLITE_TRANSIENT); + }else{ + /* sha1() - hexadecimal text result */ + hash_finish(&cx, zOut, 0); + sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); + } +} + +/* +** Implementation of the sha1_query(SQL) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using SHA1 and that hash is returned. +** +** The original SQL text is included as part of the hash. +** +** The hash is not just a concatenation of the outputs. Each query +** is delimited and each row and value within the query is delimited, +** with all values being marked with their datatypes. +*/ +static void sha1QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA1Context cx; + char zOut[44]; + + assert( argc==1 ); + if( zSql==0 ) return; + hash_init(&cx); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + if( z==0 ) z = ""; + n = (int)strlen(z); + hash_step_vformat(&cx,"S%d:",n); + hash_step(&cx,(unsigned char*)z,n); + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + hash_step(&cx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + hash_step(&cx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + hash_step(&cx,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + hash_step_vformat(&cx,"T%d:",n2); + hash_step(&cx, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + hash_step_vformat(&cx,"B%d:",n2); + hash_step(&cx, z2, n2); + break; + } + } + } + } + sqlite3_finalize(pStmt); + } + hash_finish(&cx, zOut, 0); + sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); +} + + +#ifdef _WIN32 + +#endif +int sqlite3_sha_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + static int one = 1; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "sha1", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, sha1Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha1b", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + (void*)&one, sha1Func, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha1_query", 1, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + sha1QueryFunc, 0, 0); + } + return rc; +} + +/************************* End ext/misc/sha1.c ********************/ +/************************* Begin ext/misc/uint.c ******************/ +/* +** 2020-04-14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements the UINT collating sequence. +** +** UINT works like BINARY for text, except that embedded strings +** of digits compare in numeric order. +** +** * Leading zeros are handled properly, in the sense that +** they do not mess of the magnitude comparison of embedded +** strings of digits. "x00123y" is equal to "x123y". +** +** * Only unsigned integers are recognized. Plus and minus +** signs are ignored. Decimal points and exponential notation +** are ignored. +** +** * Embedded integers can be of arbitrary length. Comparison +** is *not* limited integers that can be expressed as a +** 64-bit machine integer. +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +/* +** Compare text in lexicographic order, except strings of digits +** compare in numeric order. +*/ +static int uintCollFunc( + void *notUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + const unsigned char *zA = (const unsigned char*)pKey1; + const unsigned char *zB = (const unsigned char*)pKey2; + int i=0, j=0, x; + (void)notUsed; + while( i +#include +#include +#include + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(X) (void)(X) +#endif + +#ifndef IsSpace +#define IsSpace(X) isspace((unsigned char)X) +#endif + +#ifndef SQLITE_DECIMAL_MAX_DIGIT +# define SQLITE_DECIMAL_MAX_DIGIT 10000000 +#endif + +/* A decimal object */ +typedef struct Decimal Decimal; +struct Decimal { + char sign; /* 0 for positive, 1 for negative */ + char oom; /* True if an OOM is encountered */ + char isNull; /* True if holds a NULL rather than a number */ + char isInit; /* True upon initialization */ + int nDigit; /* Total number of digits */ + int nFrac; /* Number of digits to the right of the decimal point */ + signed char *a; /* Array of digits. Most significant first. */ +}; + +/* +** Release memory held by a Decimal, but do not free the object itself. +*/ +static void decimal_clear(Decimal *p){ + sqlite3_free(p->a); +} + +/* +** Destroy a Decimal object +*/ +static void decimal_free(Decimal *p){ + if( p ){ + decimal_clear(p); + sqlite3_free(p); + } +} + +/* +** Allocate a new Decimal object initialized to the text in zIn[]. +** Return NULL if any kind of error occurs. +*/ +static Decimal *decimalNewFromText(const char *zIn, int n){ + Decimal *p = 0; + int i; + int iExp = 0; + + if( zIn==0 ) goto new_from_text_failed; + p = sqlite3_malloc64( sizeof(*p) ); + if( p==0 ) goto new_from_text_failed; + p->sign = 0; + p->oom = 0; + p->isInit = 1; + p->isNull = 0; + p->nDigit = 0; + p->nFrac = 0; + p->a = sqlite3_malloc64( n+1 ); + if( p->a==0 ) goto new_from_text_failed; + for(i=0; IsSpace(zIn[i]); i++){} + if( zIn[i]=='-' ){ + p->sign = 1; + i++; + }else if( zIn[i]=='+' ){ + i++; + } + while( i='0' && c<='9' ){ + p->a[p->nDigit++] = c - '0'; + }else if( c=='.' ){ + p->nFrac = p->nDigit + 1; + }else if( c=='e' || c=='E' ){ + int j = i+1; + int neg = 0; + if( j>=n ) break; + if( zIn[j]=='-' ){ + neg = 1; + j++; + }else if( zIn[j]=='+' ){ + j++; + } + while( j='0' && zIn[j]<='9' ){ + iExp = iExp*10 + zIn[j] - '0'; + } + j++; + } + if( neg ) iExp = -iExp; + break; + } + i++; + } + if( p->nFrac ){ + p->nFrac = p->nDigit - (p->nFrac - 1); + } + if( iExp>0 ){ + if( p->nFrac>0 ){ + if( iExp<=p->nFrac ){ + p->nFrac -= iExp; + iExp = 0; + }else{ + iExp -= p->nFrac; + p->nFrac = 0; + } + } + if( iExp>0 ){ + signed char *a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + + (sqlite3_int64)iExp + 1 ); + if( a==0 ) goto new_from_text_failed; + p->a = a; + memset(p->a+p->nDigit, 0, iExp); + p->nDigit += iExp; + } + }else if( iExp<0 ){ + int nExtra; + iExp = -iExp; + nExtra = p->nDigit - p->nFrac - 1; + if( nExtra ){ + if( nExtra>=iExp ){ + p->nFrac += iExp; + iExp = 0; + }else{ + iExp -= nExtra; + p->nFrac = p->nDigit - 1; + } + } + if( iExp>0 ){ + signed char *a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + + (sqlite3_int64)iExp + 1 ); + if( a==0 ) goto new_from_text_failed; + p->a = a; + memmove(p->a+iExp, p->a, p->nDigit); + memset(p->a, 0, iExp); + p->nDigit += iExp; + p->nFrac += iExp; + } + } + if( p->sign ){ + for(i=0; inDigit && p->a[i]==0; i++){} + if( i>=p->nDigit ) p->sign = 0; + } + if( p->nDigit>SQLITE_DECIMAL_MAX_DIGIT ) goto new_from_text_failed; + return p; + +new_from_text_failed: + if( p ){ + if( p->a ) sqlite3_free(p->a); + sqlite3_free(p); + } + return 0; +} + +/* Forward reference */ +static Decimal *decimalFromDouble(double); + +/* +** Allocate a new Decimal object from an sqlite3_value. Return a pointer +** to the new object, or NULL if there is an error. If the pCtx argument +** is not NULL, then errors are reported on it as well. +** +** If the pIn argument is SQLITE_TEXT or SQLITE_INTEGER, it is converted +** directly into a Decimal. For SQLITE_FLOAT or for SQLITE_BLOB of length +** 8 bytes, the resulting double value is expanded into its decimal equivalent. +** If pIn is NULL or if it is a BLOB that is not exactly 8 bytes in length, +** then NULL is returned. +*/ +static Decimal *decimal_new( + sqlite3_context *pCtx, /* Report error here, if not null */ + sqlite3_value *pIn, /* Construct the decimal object from this */ + int bTextOnly /* Always interpret pIn as text if true */ +){ + Decimal *p = 0; + int eType = sqlite3_value_type(pIn); + if( bTextOnly && (eType==SQLITE_FLOAT || eType==SQLITE_BLOB) ){ + eType = SQLITE_TEXT; + } + switch( eType ){ + case SQLITE_TEXT: + case SQLITE_INTEGER: { + const char *zIn = (const char*)sqlite3_value_text(pIn); + int n = sqlite3_value_bytes(pIn); + p = decimalNewFromText(zIn, n); + if( p==0 ) goto new_failed; + break; + } + + case SQLITE_FLOAT: { + p = decimalFromDouble(sqlite3_value_double(pIn)); + break; + } + + case SQLITE_BLOB: { + const unsigned char *x; + unsigned int i; + sqlite3_uint64 v = 0; + double r; + + if( sqlite3_value_bytes(pIn)!=sizeof(r) ) break; + x = sqlite3_value_blob(pIn); + for(i=0; ioom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( p->isNull ){ + sqlite3_result_null(pCtx); + return; + } + z = sqlite3_malloc64( (sqlite3_int64)p->nDigit+4 ); + if( z==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + i = 0; + if( p->nDigit==0 || (p->nDigit==1 && p->a[0]==0) ){ + p->sign = 0; + } + if( p->sign ){ + z[0] = '-'; + i = 1; + } + n = p->nDigit - p->nFrac; + if( n<=0 ){ + z[i++] = '0'; + } + j = 0; + while( n>1 && p->a[j]==0 ){ + j++; + n--; + } + while( n>0 ){ + z[i++] = p->a[j] + '0'; + j++; + n--; + } + if( p->nFrac ){ + z[i++] = '.'; + do{ + z[i++] = p->a[j] + '0'; + j++; + }while( jnDigit ); + } + z[i] = 0; + sqlite3_result_text(pCtx, z, i, sqlite3_free); +} + +/* +** Round a decimal value to N significant digits. N must be positive. +*/ +static void decimal_round(Decimal *p, int N){ + int i; + int nZero; + if( N<1 ) return; + if( p==0 ) return; + if( p->nDigit<=N ) return; + for(nZero=0; nZeronDigit && p->a[nZero]==0; nZero++){} + N += nZero; + if( p->nDigit<=N ) return; + if( p->a[N]>4 ){ + p->a[N-1]++; + for(i=N-1; i>0 && p->a[i]>9; i--){ + p->a[i] = 0; + p->a[i-1]++; + } + if( p->a[0]>9 ){ + p->a[0] = 1; + p->nFrac--; + } + } + memset(&p->a[N], 0, p->nDigit - N); +} + +/* +** Make the given Decimal the result in an format similar to '%+#e'. +** In other words, show exponential notation with leading and trailing +** zeros omitted. +*/ +static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p, int N){ + char *z; /* The output buffer */ + int i; /* Loop counter */ + int nZero; /* Number of leading zeros */ + int nDigit; /* Number of digits not counting trailing zeros */ + int nFrac; /* Digits to the right of the decimal point */ + int exp; /* Exponent value */ + signed char zero; /* Zero value */ + signed char *a; /* Array of digits */ + + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( p->isNull ){ + sqlite3_result_null(pCtx); + return; + } + if( N<1 ) N = 0; + for(nDigit=p->nDigit; nDigit>N && p->a[nDigit-1]==0; nDigit--){} + for(nZero=0; nZeroa[nZero]==0; nZero++){} + nFrac = p->nFrac + (nDigit - p->nDigit); + nDigit -= nZero; + z = sqlite3_malloc64( (sqlite3_int64)nDigit+20 ); + if( z==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( nDigit==0 ){ + zero = 0; + a = &zero; + nDigit = 1; + nFrac = 0; + }else{ + a = &p->a[nZero]; + } + if( p->sign && nDigit>0 ){ + z[0] = '-'; + }else{ + z[0] = '+'; + } + z[1] = a[0]+'0'; + z[2] = '.'; + if( nDigit==1 ){ + z[3] = '0'; + i = 4; + }else{ + for(i=1; iisNull==0 +** pB!=0 +** pB->isNull==0 +*/ +static int decimal_cmp(Decimal *pA, Decimal *pB){ + int nASig, nBSig, rc, n; + while( pA->nFrac>0 && pA->a[pA->nDigit-1]==0 ){ + pA->nDigit--; + pA->nFrac--; + } + while( pB->nFrac>0 && pB->a[pB->nDigit-1]==0 ){ + pB->nDigit--; + pB->nFrac--; + } + if( pA->sign!=pB->sign ){ + return pA->sign ? -1 : +1; + } + if( pA->sign ){ + Decimal *pTemp = pA; + pA = pB; + pB = pTemp; + } + nASig = pA->nDigit - pA->nFrac; + nBSig = pB->nDigit - pB->nFrac; + if( nASig!=nBSig ){ + return nASig - nBSig; + } + n = pA->nDigit; + if( n>pB->nDigit ) n = pB->nDigit; + rc = memcmp(pA->a, pB->a, n); + if( rc==0 ){ + rc = pA->nDigit - pB->nDigit; + } + return rc; +} + +/* +** SQL Function: decimal_cmp(X, Y) +** +** Return negative, zero, or positive if X is less then, equal to, or +** greater than Y. +*/ +static void decimalCmpFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = 0, *pB = 0; + int rc; + + UNUSED_PARAMETER(argc); + pA = decimal_new(context, argv[0], 1); + if( pA==0 || pA->isNull ) goto cmp_done; + pB = decimal_new(context, argv[1], 1); + if( pB==0 || pB->isNull ) goto cmp_done; + rc = decimal_cmp(pA, pB); + if( rc<0 ) rc = -1; + else if( rc>0 ) rc = +1; + sqlite3_result_int(context, rc); +cmp_done: + decimal_free(pA); + decimal_free(pB); +} + +/* +** Expand the Decimal so that it has a least nDigit digits and nFrac +** digits to the right of the decimal point. +*/ +static void decimal_expand(Decimal *p, int nDigit, int nFrac){ + int nAddSig; + int nAddFrac; + signed char *a; + if( p==0 ) return; + nAddFrac = nFrac - p->nFrac; + nAddSig = (nDigit - p->nDigit) - nAddFrac; + if( nAddFrac==0 && nAddSig==0 ) return; + if( nDigit+1>SQLITE_DECIMAL_MAX_DIGIT ){ p->oom = 1; return; } + a = sqlite3_realloc64(p->a, nDigit+1); + if( a==0 ){ + p->oom = 1; + return; + } + p->a = a; + if( nAddSig ){ + memmove(p->a+nAddSig, p->a, p->nDigit); + memset(p->a, 0, nAddSig); + p->nDigit += nAddSig; + } + if( nAddFrac ){ + memset(p->a+p->nDigit, 0, nAddFrac); + p->nDigit += nAddFrac; + p->nFrac += nAddFrac; + } +} + +/* +** Add the value pB into pA. A := A + B. +** +** Both pA and pB might become denormalized by this routine. +*/ +static void decimal_add(Decimal *pA, Decimal *pB){ + int nSig, nFrac, nDigit; + int i, rc; + if( pA==0 ){ + return; + } + if( pA->oom || pB==0 || pB->oom ){ + pA->oom = 1; + return; + } + if( pA->isNull || pB->isNull ){ + pA->isNull = 1; + return; + } + nSig = pA->nDigit - pA->nFrac; + if( nSig && pA->a[0]==0 ) nSig--; + if( nSignDigit-pB->nFrac ){ + nSig = pB->nDigit - pB->nFrac; + } + nFrac = pA->nFrac; + if( nFracnFrac ) nFrac = pB->nFrac; + nDigit = nSig + nFrac + 1; + decimal_expand(pA, nDigit, nFrac); + decimal_expand(pB, nDigit, nFrac); + if( pA->oom || pB->oom ){ + pA->oom = 1; + }else{ + if( pA->sign==pB->sign ){ + int carry = 0; + for(i=nDigit-1; i>=0; i--){ + int x = pA->a[i] + pB->a[i] + carry; + if( x>=10 ){ + carry = 1; + pA->a[i] = x - 10; + }else{ + carry = 0; + pA->a[i] = x; + } + } + }else{ + signed char *aA, *aB; + int borrow = 0; + rc = memcmp(pA->a, pB->a, nDigit); + if( rc<0 ){ + aA = pB->a; + aB = pA->a; + pA->sign = !pA->sign; + }else{ + aA = pA->a; + aB = pB->a; + } + for(i=nDigit-1; i>=0; i--){ + int x = aA[i] - aB[i] - borrow; + if( x<0 ){ + pA->a[i] = x+10; + borrow = 1; + }else{ + pA->a[i] = x; + borrow = 0; + } + } + } + } +} + +/* +** Multiply A by B. A := A * B +** +** All significant digits after the decimal point are retained. +** Trailing zeros after the decimal point are omitted as long as +** the number of digits after the decimal point is no less than +** either the number of digits in either input. +*/ +static void decimalMul(Decimal *pA, Decimal *pB){ + signed char *acc = 0; + int i, j, k; + int minFrac; + sqlite3_int64 sumDigit; + + if( pA==0 || pA->oom || pA->isNull + || pB==0 || pB->oom || pB->isNull + ){ + goto mul_end; + } + sumDigit = pA->nDigit; + sumDigit += pB->nDigit; + sumDigit += 2; + if( sumDigit>SQLITE_DECIMAL_MAX_DIGIT ){ pA->oom = 1; return; } + acc = sqlite3_malloc64( sumDigit ); + if( acc==0 ){ + pA->oom = 1; + goto mul_end; + } + memset(acc, 0, pA->nDigit + pB->nDigit + 2); + minFrac = pA->nFrac; + if( pB->nFracnFrac; + for(i=pA->nDigit-1; i>=0; i--){ + signed char f = pA->a[i]; + int carry = 0, x; + for(j=pB->nDigit-1, k=i+j+3; j>=0; j--, k--){ + x = acc[k] + f*pB->a[j] + carry; + acc[k] = x%10; + carry = x/10; + } + x = acc[k] + carry; + acc[k] = x%10; + acc[k-1] += x/10; + } + sqlite3_free(pA->a); + pA->a = acc; + acc = 0; + pA->nDigit += pB->nDigit + 2; + pA->nFrac += pB->nFrac; + pA->sign ^= pB->sign; + while( pA->nFrac>minFrac && pA->a[pA->nDigit-1]==0 ){ + pA->nFrac--; + pA->nDigit--; + } + +mul_end: + sqlite3_free(acc); +} + +/* +** Create a new Decimal object that contains an integer power of 2. +*/ +static Decimal *decimalPow2(int N){ + Decimal *pA = 0; /* The result to be returned */ + Decimal *pX = 0; /* Multiplier */ + if( N<-20000 || N>20000 ) goto pow2_fault; + pA = decimalNewFromText("1.0", 3); + if( pA==0 || pA->oom ) goto pow2_fault; + if( N==0 ) return pA; + if( N>0 ){ + pX = decimalNewFromText("2.0", 3); + }else{ + N = -N; + pX = decimalNewFromText("0.5", 3); + } + if( pX==0 || pX->oom ) goto pow2_fault; + while( 1 /* Exit by break */ ){ + if( N & 1 ){ + decimalMul(pA, pX); + if( pA->oom ) goto pow2_fault; + } + N >>= 1; + if( N==0 ) break; + decimalMul(pX, pX); + } + decimal_free(pX); + return pA; + +pow2_fault: + decimal_free(pA); + decimal_free(pX); + return 0; +} + +/* +** Use an IEEE754 binary64 ("double") to generate a new Decimal object. +*/ +static Decimal *decimalFromDouble(double r){ + sqlite3_int64 m, a; + int e; + int isNeg; + Decimal *pA; + Decimal *pX; + char zNum[100]; + if( r<0.0 ){ + isNeg = 1; + r = -r; + }else{ + isNeg = 0; + } + memcpy(&a,&r,sizeof(a)); + if( a==0 || a==(sqlite3_int64)0x8000000000000000LL){ + e = 0; + m = 0; + }else{ + e = a>>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + if( e==0 ){ + m <<= 1; + }else{ + m |= ((sqlite3_int64)1)<<52; + } + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + e = e - 1075; + if( e>971 ){ + return 0; /* A NaN or an Infinity */ + } + } + + /* At this point m is the integer significand and e is the exponent */ + sqlite3_snprintf(sizeof(zNum), zNum, "%lld", m); + pA = decimalNewFromText(zNum, (int)strlen(zNum)); + pX = decimalPow2(e); + decimalMul(pA, pX); + decimal_free(pX); + return pA; +} + +/* +** SQL Function: decimal(X) +** OR: decimal_exp(X) +** +** Convert input X into decimal and then back into text. +** +** If X is originally a float, then a full decimal expansion of that floating +** point value is done. Or if X is an 8-byte blob, it is interpreted +** as a float and similarly expanded. +** +** The decimal_exp(X) function returns the result in exponential notation. +** decimal(X) returns a complete decimal, without the e+NNN at the end. +*/ +static void decimalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p = decimal_new(context, argv[0], 0); + int N; + if( argc==2 ){ + N = sqlite3_value_int(argv[1]); + if( N>0 ) decimal_round(p, N); + }else{ + N = 0; + } + if( p ){ + if( sqlite3_user_data(context)!=0 ){ + decimal_result_sci(context, p, N); + }else{ + decimal_result(context, p); + } + decimal_free(p); + } +} + +/* +** Compare text in decimal order. +*/ +static int decimalCollFunc( + void *notUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + const unsigned char *zA = (const unsigned char*)pKey1; + const unsigned char *zB = (const unsigned char*)pKey2; + Decimal *pA = decimalNewFromText((const char*)zA, nKey1); + Decimal *pB = decimalNewFromText((const char*)zB, nKey2); + int rc; + UNUSED_PARAMETER(notUsed); + if( pA==0 || pB==0 ){ + rc = 0; + }else{ + rc = decimal_cmp(pA, pB); + } + decimal_free(pA); + decimal_free(pB); + return rc; +} + + +/* +** SQL Function: decimal_add(X, Y) +** decimal_sub(X, Y) +** +** Return the sum or difference of X and Y. +*/ +static void decimalAddFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + decimal_add(pA, pB); + decimal_result(context, pA); + decimal_free(pA); + decimal_free(pB); +} +static void decimalSubFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + if( pB ){ + pB->sign = !pB->sign; + decimal_add(pA, pB); + decimal_result(context, pA); + } + decimal_free(pA); + decimal_free(pB); +} + +/* Aggregate function: decimal_sum(X) +** +** Works like sum() except that it uses decimal arithmetic for unlimited +** precision. +*/ +static void decimalSumStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p; + Decimal *pArg; + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( !p->isInit ){ + p->isInit = 1; + p->a = sqlite3_malloc64(2); + if( p->a==0 ){ + p->oom = 1; + }else{ + p->a[0] = 0; + } + p->nDigit = 1; + p->nFrac = 0; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pArg = decimal_new(context, argv[0], 1); + decimal_add(p, pArg); + decimal_free(pArg); +} +static void decimalSumInverse( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p; + Decimal *pArg; + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pArg = decimal_new(context, argv[0], 1); + if( pArg ) pArg->sign = !pArg->sign; + decimal_add(p, pArg); + decimal_free(pArg); +} +static void decimalSumValue(sqlite3_context *context){ + Decimal *p = sqlite3_aggregate_context(context, 0); + if( p==0 ) return; + decimal_result(context, p); +} +static void decimalSumFinalize(sqlite3_context *context){ + Decimal *p = sqlite3_aggregate_context(context, 0); + if( p==0 ) return; + decimal_result(context, p); + decimal_clear(p); +} + +/* +** SQL Function: decimal_mul(X, Y) +** +** Return the product of X and Y. +*/ +static void decimalMulFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + if( pA==0 || pA->oom || pA->isNull + || pB==0 || pB->oom || pB->isNull + ){ + goto mul_end; + } + decimalMul(pA, pB); + if( pA->oom ){ + goto mul_end; + } + decimal_result(context, pA); + +mul_end: + decimal_free(pA); + decimal_free(pB); +} + +/* +** SQL Function: decimal_pow2(N) +** +** Return the N-th power of 2. N must be an integer. +*/ +static void decimalPow2Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ + Decimal *pA = decimalPow2(sqlite3_value_int(argv[0])); + decimal_result_sci(context, pA, 0); + decimal_free(pA); + } +} + +#ifdef _WIN32 + +#endif +int sqlite3_decimal_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + static const struct { + const char *zFuncName; + int nArg; + int iArg; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "decimal", 1, 0, decimalFunc }, + { "decimal", 2, 0, decimalFunc }, + { "decimal_exp", 1, 1, decimalFunc }, + { "decimal_exp", 2, 1, decimalFunc }, + { "decimal_cmp", 2, 0, decimalCmpFunc }, + { "decimal_add", 2, 0, decimalAddFunc }, + { "decimal_sub", 2, 0, decimalSubFunc }, + { "decimal_mul", 2, 0, decimalMulFunc }, + { "decimal_pow2", 1, 0, decimalPow2Func }, + }; + unsigned int i; + (void)pzErrMsg; /* Unused parameter */ + + SQLITE_EXTENSION_INIT2(pApi); + + for(i=0; i<(int)(sizeof(aFunc)/sizeof(aFunc[0])) && rc==SQLITE_OK; i++){ + rc = sqlite3_create_function(db, aFunc[i].zFuncName, aFunc[i].nArg, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + aFunc[i].iArg ? db : 0, aFunc[i].xFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_window_function(db, "decimal_sum", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, 0, + decimalSumStep, decimalSumFinalize, + decimalSumValue, decimalSumInverse, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_collation(db, "decimal", SQLITE_UTF8, + 0, decimalCollFunc); + } + return rc; +} + +/************************* End ext/misc/decimal.c ********************/ +/************************* Begin ext/misc/base64.c ******************/ +/* +** 2022-11-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a SQLite extension for converting in either direction +** between a (binary) blob and base64 text. Base64 can transit a +** sane USASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals, +** and can be used unmodified in XML-like documents. +** +** This is an independent implementation of conversions specified in +** RFC 4648, done on the above date by the author (Larry Brasfield) +** who thereby has the right to put this into the public domain. +** +** The conversions meet RFC 4648 requirements, provided that this +** C source specifies that line-feeds are included in the encoded +** data to limit visible line lengths to 72 characters and to +** terminate any encoded blob having non-zero length. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Generated base64 sequences, with their line-feeds included, +** can be concatenated; the result converted back to binary will +** be the concatenation of the represented binary sequences. +** +** This SQLite3 extension creates a function, base64(x), which +** either: converts text x containing base64 to a returned blob; +** or converts a blob x to returned text containing base64. An +** error will be thrown for other input argument types. +** +** This code relies on UTF-8 encoding only with respect to the +** meaning of the first 128 (7-bit) codes matching that of USASCII. +** It will fail miserably if somehow made to try to convert EBCDIC. +** Because it is table-driven, it could be enhanced to handle that, +** but the world and SQLite have moved on from that anachronism. +** +** To build the extension: +** Set shell variable SQDIR= +** *Nix: gcc -O2 -shared -I$SQDIR -fPIC -o base64.so base64.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR -o base64.dylib base64.c +** Win32: gcc -O2 -shared -I%SQDIR% -o base64.dll base64.c +** Win32: cl /Os -I%SQDIR% base64.c -link -dll -out:base64.dll +*/ + +#include + +/* #include "sqlite3ext.h" */ + +#ifndef deliberate_fall_through +/* Quiet some compilers about some of our intentional code. */ +# if GCC_VERSION>=7000000 +# define deliberate_fall_through __attribute__((fallthrough)); +# else +# define deliberate_fall_through +# endif +#endif + +SQLITE_EXTENSION_INIT1; + +#define PC 0x80 /* pad character */ +#define WS 0x81 /* whitespace */ +#define ND 0x82 /* Not above or digit-value */ +#define PAD_CHAR '=' + +#ifndef U8_TYPEDEF +/* typedef unsigned char u8; */ +#define U8_TYPEDEF +#endif + +/* Decoding table, ASCII (7-bit) value to base 64 digit value or other */ +static const u8 b64DigitValues[128] = { + /* HT LF VT FF CR */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,WS,WS,WS, WS,WS,ND,ND, + /* US */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, + /*sp + / */ + WS,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,62, ND,ND,ND,63, + /* 0 1 5 9 = */ + 52,53,54,55, 56,57,58,59, 60,61,ND,ND, ND,PC,ND,ND, + /* A O */ + ND, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, + /* P Z */ + 15,16,17,18, 19,20,21,22, 23,24,25,ND, ND,ND,ND,ND, + /* a o */ + ND,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, + /* p z */ + 41,42,43,44, 45,46,47,48, 49,50,51,ND, ND,ND,ND,ND +}; + +static const char b64Numerals[64+1] += "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#define BX_DV_PROTO(c) \ + ((((u8)(c))<0x80)? (u8)(b64DigitValues[(u8)(c)]) : 0x80) +#define IS_BX_DIGIT(bdp) (((u8)(bdp))<0x80) +#define IS_BX_WS(bdp) ((bdp)==WS) +#define IS_BX_PAD(bdp) ((bdp)==PC) +#define BX_NUMERAL(dv) (b64Numerals[(u8)(dv)]) +/* Width of base64 lines. Should be an integer multiple of 4. */ +#define B64_DARK_MAX 72 + +/* Encode a byte buffer into base64 text with linefeeds appended to limit +** encoded group lengths to B64_DARK_MAX or to terminate the last group. +*/ +static char* toBase64( u8 *pIn, int nbIn, char *pOut ){ + int nCol = 0; + while( nbIn >= 3 ){ + /* Do the bit-shuffle, exploiting unsigned input to avoid masking. */ + pOut[0] = BX_NUMERAL(pIn[0]>>2); + pOut[1] = BX_NUMERAL(((pIn[0]<<4)|(pIn[1]>>4))&0x3f); + pOut[2] = BX_NUMERAL(((pIn[1]&0xf)<<2)|(pIn[2]>>6)); + pOut[3] = BX_NUMERAL(pIn[2]&0x3f); + pOut += 4; + nbIn -= 3; + pIn += 3; + if( (nCol += 4)>=B64_DARK_MAX || nbIn<=0 ){ + *pOut++ = '\n'; + nCol = 0; + } + } + if( nbIn > 0 ){ + signed char nco = nbIn+1; + int nbe; + unsigned long qv = *pIn++; + for( nbe=1; nbe<3; ++nbe ){ + qv <<= 8; + if( nbe=0; --nbe ){ + char ce = (nbe>= 6; + pOut[nbe] = ce; + } + pOut += 4; + *pOut++ = '\n'; + } + *pOut = 0; + return pOut; +} + +/* Skip over text which is not base64 numeral(s). */ +static char * skipNonB64( char *s, int nc ){ + char c; + while( nc-- > 0 && (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s; + return s; +} + +/* Decode base64 text into a byte buffer. */ +static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 && *pIn!=PAD_CHAR ){ + static signed char nboi[] = { 0, 0, 1, 2, 3 }; + char *pUse = skipNonB64(pIn, ncIn); + unsigned long qv = 0L; + int nti, nbo, nac; + ncIn -= (pUse - pIn); + pIn = pUse; + nti = (ncIn>4)? 4 : ncIn; + ncIn -= nti; + nbo = nboi[nti]; + if( nbo==0 ) break; + for( nac=0; nac<4; ++nac ){ + char c = (nac>8) & 0xff; + deliberate_fall_through; /* FALLTHRU */ + case 1: + pOut[0] = (qv>>16) & 0xff; + break; + } + pOut += nbo; + } + return pOut; +} + +/* This function does the work for the SQLite base64(x) UDF. */ +static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ + sqlite3_int64 nb; + sqlite3_int64 nv = sqlite3_value_bytes(av[0]); + sqlite3_int64 nc; + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + u8 *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + nc = 4*((nv+2)/3); /* quads needed */ + nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base64 too big", -1); + return; + } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } + cBuf = sqlite3_malloc64(nc); + if( !cBuf ) goto memFail; + nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 3*((nv+3)/4); /* may overestimate due to LF and padding */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base64 may be too big", -1); + return; + }else if( nb<1 ){ + nb = 1; + } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } + bBuf = sqlite3_malloc64(nb); + if( !bBuf ) goto memFail; + nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base64 accepts only blob or text", -1); + return; + } + return; + memFail: + sqlite3_result_error(context, "base64 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS +#ifdef _WIN32 + +#endif +int sqlite3_base64_init +#else +static int sqlite3_base64_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; + return sqlite3_create_function + (db, "base64", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base64, 0, 0); +} + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +#define BASE64_INIT(db) sqlite3_base64_init(db, 0, 0) +#define BASE64_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ + +/************************* End ext/misc/base64.c ********************/ +/************************* Begin ext/misc/base85.c ******************/ +/* +** 2022-11-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility for converting binary to base85 or vice-versa. +** It can be built as a standalone program or an SQLite3 extension. +** +** Much like base64 representations, base85 can be sent through a +** sane USASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals. +** It is not suited for unmodified use in XML-like documents. +** +** The encoding used resembles Ascii85, but was devised by the author +** (Larry Brasfield) before Mozilla, Adobe, ZMODEM or other Ascii85 +** variant sources existed, in the 1984 timeframe on a VAX mainframe. +** Further, this is an independent implementation of a base85 system. +** Hence, the author has rightfully put this into the public domain. +** +** Base85 numerals are taken from the set of 7-bit USASCII codes, +** excluding control characters and Space ! " ' ( ) { | } ~ Del +** in code order representing digit values 0 to 84 (base 10.) +** +** Groups of 4 bytes, interpreted as big-endian 32-bit values, +** are represented as 5-digit base85 numbers with MS to LS digit +** order. Groups of 1-3 bytes are represented with 2-4 digits, +** still big-endian but 8-24 bit values. (Using big-endian yields +** the simplest transition to byte groups smaller than 4 bytes. +** These byte groups can also be considered base-256 numbers.) +** Groups of 0 bytes are represented with 0 digits and vice-versa. +** No pad characters are used; Encoded base85 numeral sequence +** (aka "group") length maps 1-to-1 to the decoded binary length. +** +** Any character not in the base85 numeral set delimits groups. +** When base85 is streamed or stored in containers of indefinite +** size, newline is used to separate it into sub-sequences of no +** more than 80 digits so that fgets() can be used to read it. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Base85 sequences can be concatenated by separating them with +** a non-base85 character; the conversion to binary will then +** be the concatenation of the represented binary sequences. + +** The standalone program either converts base85 on stdin to create +** a binary file or converts a binary file to base85 on stdout. +** Read or make it blurt its help for invocation details. +** +** The SQLite3 extension creates a function, base85(x), which will +** either convert text base85 to a blob or a blob to text base85 +** and return the result (or throw an error for other types.) +** Unless built with OMIT_BASE85_CHECKER defined, it also creates a +** function, is_base85(t), which returns 1 iff the text t contains +** nothing other than base85 numerals and whitespace, or 0 otherwise. +** +** To build the extension: +** Set shell variable SQDIR= +** and variable OPTS to -DOMIT_BASE85_CHECKER if is_base85() unwanted. +** *Nix: gcc -O2 -shared -I$SQDIR $OPTS -fPIC -o base85.so base85.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR $OPTS -o base85.dylib base85.c +** Win32: gcc -O2 -shared -I%SQDIR% %OPTS% -o base85.dll base85.c +** Win32: cl /Os -I%SQDIR% %OPTS% base85.c -link -dll -out:base85.dll +** +** To build the standalone program, define PP symbol BASE85_STANDALONE. Eg. +** *Nix or OSX: gcc -O2 -DBASE85_STANDALONE base85.c -o base85 +** Win32: gcc -O2 -DBASE85_STANDALONE -o base85.exe base85.c +** Win32: cl /Os /MD -DBASE85_STANDALONE base85.c +*/ + +#include +#include +#include +#include +#ifndef OMIT_BASE85_CHECKER +# include +#endif + +#ifndef BASE85_STANDALONE + +/* # include "sqlite3ext.h" */ + +SQLITE_EXTENSION_INIT1; + +#else + +# ifdef _WIN32 +# include +# include +# else +# define setmode(fd,m) +# endif + +static char *zHelp = + "Usage: base85 \n" + " is either -r to read or -w to write ,\n" + " content to be converted to/from base85 on stdout/stdin.\n" + " names a binary file to be rendered or created.\n" + " Or, the name '-' refers to the stdin or stdout stream.\n" + ; + +static void sayHelp(){ + printf("%s", zHelp); +} +#endif + +#ifndef U8_TYPEDEF +/* typedef unsigned char u8; */ +#define U8_TYPEDEF +#endif + +/* Classify c according to interval within USASCII set w.r.t. base85 + * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. + */ +#define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z')) + +/* Provide digitValue to b85Numeral offset as a function of above class. */ +static u8 b85_cOffset[] = { 0, '#', 0, '*'-4, 0 }; +#define B85_DNOS( c ) b85_cOffset[B85_CLASS(c)] + +/* Say whether c is a base85 numeral. */ +#define IS_B85( c ) (B85_CLASS(c) & 1) + +#if 0 /* Not used, */ +static u8 base85DigitValue( char c ){ + u8 dv = (u8)(c - '#'); + if( dv>87 ) return 0xff; + return (dv > 3)? dv-3 : dv; +} +#endif + +/* Width of base64 lines. Should be an integer multiple of 5. */ +#define B85_DARK_MAX 80 + + +static char * skipNonB85( char *s, int nc ){ + char c; + while( nc-- > 0 && (c = *s) && !IS_B85(c) ) ++s; + return s; +} + +/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral. + * Do not use the macro form with argument expression having a side-effect.*/ +#if 0 +static char base85Numeral( u8 b ){ + return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); +} +#else +# define base85Numeral( dn )\ + ((char)(((dn) < 4)? (char)((dn) + '#') : (char)((dn) - 4 + '*'))) +#endif + +static char *putcs(char *pc, char *s){ + char c; + while( (c = *s++)!=0 ) *pc++ = c; + return pc; +} + +/* Encode a byte buffer into base85 text. If pSep!=0, it's a C string +** to be appended to encoded groups to limit their length to B85_DARK_MAX +** or to terminate the last group (to aid concatenation.) +*/ +static char* toBase85( u8 *pIn, int nbIn, char *pOut, char *pSep ){ + int nCol = 0; + while( nbIn >= 4 ){ + int nco = 5; + unsigned long qbv = (((unsigned long)pIn[0])<<24) | + (pIn[1]<<16) | (pIn[2]<<8) | pIn[3]; + while( nco > 0 ){ + unsigned nqv = (unsigned)(qbv/85UL); + unsigned char dv = qbv - 85UL*nqv; + qbv = nqv; + pOut[--nco] = base85Numeral(dv); + } + nbIn -= 4; + pIn += 4; + pOut += 5; + if( pSep && (nCol += 5)>=B85_DARK_MAX ){ + pOut = putcs(pOut, pSep); + nCol = 0; + } + } + if( nbIn > 0 ){ + int nco = nbIn + 1; + unsigned long qv = *pIn++; + int nbe = 1; + while( nbe++ < nbIn ){ + qv = (qv<<8) | *pIn++; + } + nCol += nco; + while( nco > 0 ){ + u8 dv = (u8)(qv % 85); + qv /= 85; + pOut[--nco] = base85Numeral(dv); + } + pOut += (nbIn+1); + } + if( pSep && nCol>0 ) pOut = putcs(pOut, pSep); + *pOut = 0; + return pOut; +} + +/* Decode base85 text into a byte buffer. */ +static u8* fromBase85( char *pIn, int ncIn, u8 *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 ){ + static signed char nboi[] = { 0, 0, 1, 2, 3, 4 }; + char *pUse = skipNonB85(pIn, ncIn); + unsigned long qv = 0L; + int nti, nbo; + ncIn -= (pUse - pIn); + pIn = pUse; + nti = (ncIn>5)? 5 : ncIn; + nbo = nboi[nti]; + if( nbo==0 ) break; + while( nti>0 ){ + char c = *pIn++; + u8 cdo = B85_DNOS(c); + --ncIn; + if( cdo==0 ) break; + qv = 85 * qv + (c - cdo); + --nti; + } + nbo -= nti; /* Adjust for early (non-digit) end of group. */ + switch( nbo ){ + case 4: + *pOut++ = (qv >> 24)&0xff; + /* FALLTHRU */ + case 3: + *pOut++ = (qv >> 16)&0xff; + /* FALLTHRU */ + case 2: + *pOut++ = (qv >> 8)&0xff; + /* FALLTHRU */ + case 1: + *pOut++ = qv&0xff; + /* FALLTHRU */ + case 0: + break; + } + } + return pOut; +} + +#ifndef OMIT_BASE85_CHECKER +/* Say whether input char sequence is all (base85 and/or whitespace).*/ +static int allBase85( char *p, int len ){ + char c; + while( len-- > 0 && (c = *p++) != 0 ){ + if( !IS_B85(c) && !isspace(c) ) return 0; + } + return 1; +} +#endif + +#ifndef BASE85_STANDALONE + +#ifndef OMIT_BASE85_CHECKER +/* This function does the work for the SQLite is_base85(t) UDF. */ +static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_TEXT: + { + int rv = allBase85( (char *)sqlite3_value_text(av[0]), + sqlite3_value_bytes(av[0]) ); + sqlite3_result_int(context, rv); + } + break; + case SQLITE_NULL: + sqlite3_result_null(context); + break; + default: + sqlite3_result_error(context, "is_base85 accepts only text or NULL", -1); + return; + } +} +#endif + +/* This function does the work for the SQLite base85(x) UDF. */ +static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + sqlite3_int64 nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + u8 *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + /* ulongs tail newlines tailenc+nul*/ + nc = 5*(nv/4) + nv%4 + nv/64+1 + 2; + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base85 too big", -1); + return; + } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } + cBuf = sqlite3_malloc64(nc); + if( !cBuf ) goto memFail; + nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 4*(nv/5) + nv%5; /* may overestimate */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base85 may be too big", -1); + return; + }else if( nb<1 ){ + nb = 1; + } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } + bBuf = sqlite3_malloc64(nb); + if( !bBuf ) goto memFail; + nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base85 accepts only blob or text.", -1); + return; + } + return; + memFail: + sqlite3_result_error(context, "base85 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS +#ifdef _WIN32 + +#endif +int sqlite3_base85_init +#else +static int sqlite3_base85_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; +#ifndef OMIT_BASE85_CHECKER + { + int rc = sqlite3_create_function + (db, "is_base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_UTF8, + 0, is_base85, 0, 0); + if( rc!=SQLITE_OK ) return rc; + } +#endif + return sqlite3_create_function + (db, "base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base85, 0, 0); +} + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +# define BASE85_INIT(db) sqlite3_base85_init(db, 0, 0) +# define BASE85_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ + +#else /* standalone program */ + +int main(int na, char *av[]){ + int cin; + int rc = 0; + u8 bBuf[4*(B85_DARK_MAX/5)]; + char cBuf[5*(sizeof(bBuf)/4)+2]; + size_t nio; +# ifndef OMIT_BASE85_CHECKER + int b85Clean = 1; +# endif + char rw; + FILE *fb = 0, *foc = 0; + char fmode[3] = "xb"; + if( na < 3 || av[1][0]!='-' || (rw = av[1][1])==0 || (rw!='r' && rw!='w') ){ + sayHelp(); + return 0; + } + fmode[0] = rw; + if( av[2][0]=='-' && av[2][1]==0 ){ + switch( rw ){ + case 'r': + fb = stdin; + setmode(fileno(stdin), O_BINARY); + break; + case 'w': + fb = stdout; + setmode(fileno(stdout), O_BINARY); + break; + } + }else{ + fb = fopen(av[2], fmode); + foc = fb; + } + if( !fb ){ + fprintf(stderr, "Cannot open %s for %c\n", av[2], rw); + rc = 1; + }else{ + switch( rw ){ + case 'r': + while( (nio = fread( bBuf, 1, sizeof(bBuf), fb))>0 ){ + toBase85( bBuf, (int)nio, cBuf, 0 ); + fprintf(stdout, "%s\n", cBuf); + } + break; + case 'w': + while( 0 != fgets(cBuf, sizeof(cBuf), stdin) ){ + int nc = strlen(cBuf); + size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf; + if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1; +#ifndef OMIT_BASE85_CHECKER + b85Clean &= allBase85( cBuf, nc ); +#endif + } + break; + default: + sayHelp(); + rc = 1; + } + if( foc ) fclose(foc); + } +# ifndef OMIT_BASE85_CHECKER + if( !b85Clean ){ + fprintf(stderr, "Base85 input had non-base85 dark or control content.\n"); + } +# endif + return rc; +} + +#endif + +/************************* End ext/misc/base85.c ********************/ +/************************* Begin ext/misc/ieee754.c ******************/ +/* +** 2013-04-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements functions for the exact display +** and input of IEEE754 Binary64 floating-point numbers. +** +** ieee754(X) +** ieee754(Y,Z) +** +** In the first form, the value X should be a floating-point number. +** The function will return a string of the form 'ieee754(Y,Z)' where +** Y and Z are integers such that X==Y*pow(2,Z). +** +** In the second form, Y and Z are integers which are the mantissa and +** base-2 exponent of a new floating point number. The function returns +** a floating-point value equal to Y*pow(2,Z). +** +** Examples: +** +** ieee754(2.0) -> 'ieee754(2,0)' +** ieee754(45.25) -> 'ieee754(181,-2)' +** ieee754(2, 0) -> 2.0 +** ieee754(181, -2) -> 45.25 +** +** Two additional functions break apart the one-argument ieee754() +** result into separate integer values: +** +** ieee754_mantissa(45.25) -> 181 +** ieee754_exponent(45.25) -> -2 +** +** These functions convert binary64 numbers into blobs and back again. +** +** ieee754_from_blob(x'3ff0000000000000') -> 1.0 +** ieee754_to_blob(1.0) -> x'3ff0000000000000' +** +** In all single-argument functions, if the argument is an 8-byte blob +** then that blob is interpreted as a big-endian binary64 value. +** +** +** EXACT DECIMAL REPRESENTATION OF BINARY64 VALUES +** ----------------------------------------------- +** +** This extension in combination with the separate 'decimal' extension +** can be used to compute the exact decimal representation of binary64 +** values. To begin, first compute a table of exponent values: +** +** CREATE TABLE pow2(x INTEGER PRIMARY KEY, v TEXT); +** WITH RECURSIVE c(x,v) AS ( +** VALUES(0,'1') +** UNION ALL +** SELECT x+1, decimal_mul(v,'2') FROM c WHERE x+1<=971 +** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** WITH RECURSIVE c(x,v) AS ( +** VALUES(-1,'0.5') +** UNION ALL +** SELECT x-1, decimal_mul(v,'0.5') FROM c WHERE x-1>=-1075 +** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** +** Then, to compute the exact decimal representation of a floating +** point value (the value 47.49 is used in the example) do: +** +** WITH c(n) AS (VALUES(47.49)) +** ---------------^^^^^---- Replace with whatever you want +** SELECT decimal_mul(ieee754_mantissa(c.n),pow2.v) +** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.n); +** +** Here is a query to show various boundry values for the binary64 +** number format: +** +** WITH c(name,bin) AS (VALUES +** ('minimum positive value', x'0000000000000001'), +** ('maximum subnormal value', x'000fffffffffffff'), +** ('minimum positive normal value', x'0010000000000000'), +** ('maximum value', x'7fefffffffffffff')) +** SELECT c.name, decimal_mul(ieee754_mantissa(c.bin),pow2.v) +** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.bin); +** +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(X) (void)(X) +#endif + +/* +** Implementation of the ieee754() function +*/ +static void ieee754func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + if( argc==1 ){ + sqlite3_int64 m, a; + double r; + int e; + int isNeg; + char zResult[100]; + assert( sizeof(m)==sizeof(r) ); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + && sqlite3_value_bytes(argv[0])==sizeof(r) + ){ + const unsigned char *x = sqlite3_value_blob(argv[0]); + unsigned int i; + sqlite3_uint64 v = 0; + for(i=0; i>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + if( e==0 ){ + m <<= 1; + }else{ + m |= ((sqlite3_int64)1)<<52; + } + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + } + switch( *(int*)sqlite3_user_data(context) ){ + case 0: + sqlite3_snprintf(sizeof(zResult), zResult, "ieee754(%lld,%d)", + m, e-1075); + sqlite3_result_text(context, zResult, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_int64(context, m); + break; + case 2: + sqlite3_result_int(context, e-1075); + break; + } + }else{ + sqlite3_int64 m, e, a; + double r; + int isNeg = 0; + m = sqlite3_value_int64(argv[0]); + e = sqlite3_value_int64(argv[1]); + + /* Limit the range of e. Ticket 22dea1cfdb9151e4 2021-03-02 */ + if( e>10000 ){ + e = 10000; + }else if( e<-10000 ){ + e = -10000; + } + + if( m<0 ){ + if( m<(-9223372036854775807LL) ) return; + isNeg = 1; + m = -m; + }else if( m==0 && e>-1000 && e<1000 ){ + sqlite3_result_double(context, 0.0); + return; + } + while( (m>>32)&0xffe00000 ){ + m >>= 1; + e++; + } + while( m!=0 && ((m>>32)&0xfff00000)==0 ){ + m <<= 1; + e--; + } + e += 1075; + if( e<=0 ){ + /* Subnormal */ + if( 1-e >= 64 ){ + m = 0; + }else{ + m >>= 1-e; + } + e = 0; + }else if( e>0x7ff ){ + e = 0x7ff; + } + a = m & ((((sqlite3_int64)1)<<52)-1); + a |= e<<52; + if( isNeg ) a |= ((sqlite3_uint64)1)<<63; + memcpy(&r, &a, sizeof(r)); + sqlite3_result_double(context, r); + } +} + +/* +** Functions to convert between blobs and floats. +*/ +static void ieee754func_from_blob( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + && sqlite3_value_bytes(argv[0])==sizeof(double) + ){ + double r; + const unsigned char *x = sqlite3_value_blob(argv[0]); + unsigned int i; + sqlite3_uint64 v = 0; + for(i=0; i>= 8; + } + sqlite3_result_blob(context, a, sizeof(r), SQLITE_TRANSIENT); + } +} + +/* +** Functions to convert between 64-bit integers and floats. +** +** The bit patterns are copied. The numeric values are different. +*/ +static void ieee754func_from_int( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ + double r; + sqlite3_int64 v = sqlite3_value_int64(argv[0]); + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(context, r); + } +} +static void ieee754func_to_int( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_FLOAT ){ + double r = sqlite3_value_double(argv[0]); + sqlite3_uint64 v; + memcpy(&v, &r, sizeof(v)); + sqlite3_result_int64(context, v); + } +} + +/* +** SQL Function: ieee754_inc(r,N) +** +** Move the floating point value r by N quantums and return the new +** values. +** +** Behind the scenes: this routine merely casts r into a 64-bit unsigned +** integer, adds N, then casts the value back into float. +** +** Example: To find the smallest positive number: +** +** SELECT ieee754_inc(0.0,+1); +*/ +static void ieee754inc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + double r; + sqlite3_int64 N; + sqlite3_uint64 m1, m2; + double r2; + UNUSED_PARAMETER(argc); + r = sqlite3_value_double(argv[0]); + N = sqlite3_value_int64(argv[1]); + memcpy(&m1, &r, 8); + m2 = m1 + N; + memcpy(&r2, &m2, 8); + sqlite3_result_double(context, r2); +} + + +#ifdef _WIN32 + +#endif +int sqlite3_ieee_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + static const struct { + char *zFName; + int nArg; + int iAux; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "ieee754", 1, 0, ieee754func }, + { "ieee754", 2, 0, ieee754func }, + { "ieee754_mantissa", 1, 1, ieee754func }, + { "ieee754_exponent", 1, 2, ieee754func }, + { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, + { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, + { "ieee754_to_int", 1, 0, ieee754func_to_int }, + { "ieee754_from_int", 1, 0, ieee754func_from_int }, + { "ieee754_inc", 2, 0, ieee754inc }, + }; + unsigned int i; + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + for(i=0; i= 0 ) +** for each produced value (independent of production time ordering.) +** +** All parameters must be either integer or convertable to integer. +** The start parameter is required. +** The stop parameter defaults to (1<<32)-1 (aka 4294967295 or 0xffffffff) +** The step parameter defaults to 1 and 0 is treated as 1. +** +** Examples: +** +** SELECT * FROM generate_series(0,100,5); +** +** The query above returns integers from 0 through 100 counting by steps +** of 5. In other words, 0, 5, 10, 15, ..., 90, 95, 100. There are a total +** of 21 rows. +** +** SELECT * FROM generate_series(0,100); +** +** Integers from 0 through 100 with a step size of 1. 101 rows. +** +** SELECT * FROM generate_series(20) LIMIT 10; +** +** Integers 20 through 29. 10 rows. +** +** SELECT * FROM generate_series(0,-100,-5); +** +** Integers 0 -5 -10 ... -100. 21 rows. +** +** SELECT * FROM generate_series(0,-1); +** +** Empty sequence. +** +** HOW IT WORKS +** +** The generate_series "function" is really a virtual table with the +** following schema: +** +** CREATE TABLE generate_series( +** value, +** start HIDDEN, +** stop HIDDEN, +** step HIDDEN +** ); +** +** The virtual table also has a rowid which is an alias for the value. +** +** Function arguments in queries against this virtual table are translated +** into equality constraints against successive hidden columns. In other +** words, the following pairs of queries are equivalent to each other: +** +** SELECT * FROM generate_series(0,100,5); +** SELECT * FROM generate_series WHERE start=0 AND stop=100 AND step=5; +** +** SELECT * FROM generate_series(0,100); +** SELECT * FROM generate_series WHERE start=0 AND stop=100; +** +** SELECT * FROM generate_series(20) LIMIT 10; +** SELECT * FROM generate_series WHERE start=20 LIMIT 10; +** +** The generate_series virtual table implementation leaves the xCreate method +** set to NULL. This means that it is not possible to do a CREATE VIRTUAL +** TABLE command with "generate_series" as the USING argument. Instead, there +** is a single generate_series virtual table that is always available without +** having to be created first. +** +** The xBestIndex method looks for equality constraints against the hidden +** start, stop, and step columns, and if present, it uses those constraints +** to bound the sequence of generated values. If the equality constraints +** are missing, it uses 0 for start, 4294967295 for stop, and 1 for step. +** xBestIndex returns a small cost when both start and stop are available, +** and a very large cost if either start or stop are unavailable. This +** encourages the query planner to order joins such that the bounds of the +** series are well-defined. +** +** Update on 2024-08-22: +** xBestIndex now also looks for equality and inequality constraints against +** the value column and uses those constraints as additional bounds against +** the sequence range. Thus, a query like this: +** +** SELECT value FROM generate_series($SA,$EA) +** WHERE value BETWEEN $SB AND $EB; +** +** Is logically the same as: +** +** SELECT value FROM generate_series(max($SA,$SB),min($EA,$EB)); +** +** Constraints on the value column can server as substitutes for constraints +** on the hidden start and stop columns. So, the following two queries +** are equivalent: +** +** SELECT value FROM generate_series($S,$E); +** SELECT value FROM generate_series WHERE value BETWEEN $S and $E; +** +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* series_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result. +** +** iOBase, iOTerm, and iOStep are the original values of the +** start=, stop=, and step= constraints on the query. These are +** the values reported by the start, stop, and step columns of the +** virtual table. +** +** iBase, iTerm, iStep, and bDescp are the actual values used to generate +** the sequence. These might be different from the iOxxxx values. +** For example in +** +** SELECT value FROM generate_series(1,11,2) +** WHERE value BETWEEN 4 AND 8; +** +** The iOBase is 1, but the iBase is 5. iOTerm is 11 but iTerm is 7. +** Another example: +** +** SELECT value FROM generate_series(1,15,3) ORDER BY value DESC; +** +** The cursor initialization for the above query is: +** +** iOBase = 1 iBase = 13 +** iOTerm = 15 iTerm = 1 +** iOStep = 3 iStep = 3 bDesc = 1 +** +** The actual step size is unsigned so that can have a value of +** +9223372036854775808 which is needed for querys like this: +** +** SELECT value +** FROM generate_series(9223372036854775807, +** -9223372036854775808, +** -9223372036854775808) +** ORDER BY value ASC; +** +** The setup for the previous query will be: +** +** iOBase = 9223372036854775807 iBase = -1 +** iOTerm = -9223372036854775808 iTerm = 9223372036854775807 +** iOStep = -9223372036854775808 iStep = 9223372036854775808 bDesc = 0 +*/ +/* typedef unsigned char u8; */ +typedef struct series_cursor series_cursor; +struct series_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iOBase; /* Original starting value ("start") */ + sqlite3_int64 iOTerm; /* Original terminal value ("stop") */ + sqlite3_int64 iOStep; /* Original step value */ + sqlite3_int64 iBase; /* Starting value to actually use */ + sqlite3_int64 iTerm; /* Terminal value to actually use */ + sqlite3_uint64 iStep; /* The step size */ + sqlite3_int64 iValue; /* Current value */ + u8 bDesc; /* iStep is really negative */ + u8 bDone; /* True if stepped past last element */ +}; + +/* +** Computed the difference between two 64-bit signed integers using a +** convoluted computation designed to work around the silly restriction +** against signed integer overflow in C. +*/ +static sqlite3_uint64 span64(sqlite3_int64 a, sqlite3_int64 b){ + assert( a>=b ); + return (*(sqlite3_uint64*)&a) - (*(sqlite3_uint64*)&b); +} + +/* +** Add or substract an unsigned 64-bit integer from a signed 64-bit integer +** and return the new signed 64-bit integer. +*/ +static sqlite3_int64 add64(sqlite3_int64 a, sqlite3_uint64 b){ + sqlite3_uint64 x = *(sqlite3_uint64*)&a; + x += b; + return *(sqlite3_int64*)&x; +} +static sqlite3_int64 sub64(sqlite3_int64 a, sqlite3_uint64 b){ + sqlite3_uint64 x = *(sqlite3_uint64*)&a; + x -= b; + return *(sqlite3_int64*)&x; +} + +/* +** The seriesConnect() method is invoked to create a new +** series_vtab that describes the generate_series virtual table. +** +** Think of this routine as the constructor for series_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the series_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against generate_series will look like. +*/ +static int seriesConnect( + sqlite3 *db, + void *pUnused, + int argcUnused, const char *const*argvUnused, + sqlite3_vtab **ppVtab, + char **pzErrUnused +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define SERIES_COLUMN_ROWID (-1) +#define SERIES_COLUMN_VALUE 0 +#define SERIES_COLUMN_START 1 +#define SERIES_COLUMN_STOP 2 +#define SERIES_COLUMN_STEP 3 + + (void)pUnused; + (void)argcUnused; + (void)argvUnused; + (void)pzErrUnused; + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(value,start hidden,stop hidden,step hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc64( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + return rc; +} + +/* +** This method is the destructor for series_cursor objects. +*/ +static int seriesDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new series_cursor object. +*/ +static int seriesOpen(sqlite3_vtab *pUnused, sqlite3_vtab_cursor **ppCursor){ + series_cursor *pCur; + (void)pUnused; + pCur = sqlite3_malloc64( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a series_cursor. +*/ +static int seriesClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a series_cursor to its next row of output. +*/ +static int seriesNext(sqlite3_vtab_cursor *cur){ + series_cursor *pCur = (series_cursor*)cur; + if( pCur->iValue==pCur->iTerm ){ + pCur->bDone = 1; + }else if( pCur->bDesc ){ + pCur->iValue = sub64(pCur->iValue, pCur->iStep); + assert( pCur->iValue>=pCur->iTerm ); + }else{ + pCur->iValue = add64(pCur->iValue, pCur->iStep); + assert( pCur->iValue<=pCur->iTerm ); + } + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the series_cursor +** is currently pointing. +*/ +static int seriesColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + series_cursor *pCur = (series_cursor*)cur; + sqlite3_int64 x = 0; + switch( i ){ + case SERIES_COLUMN_START: x = pCur->iOBase; break; + case SERIES_COLUMN_STOP: x = pCur->iOTerm; break; + case SERIES_COLUMN_STEP: x = pCur->iOStep; break; + default: x = pCur->iValue; break; + } + sqlite3_result_int64(ctx, x); + return SQLITE_OK; +} + +#ifndef LARGEST_UINT64 +#define LARGEST_INT64 ((sqlite3_int64)0x7fffffffffffffffLL) +#define LARGEST_UINT64 ((sqlite3_uint64)0xffffffffffffffffULL) +#define SMALLEST_INT64 ((sqlite3_int64)0x8000000000000000LL) +#endif + +/* +** The rowid is the same as the value. +*/ +static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + series_cursor *pCur = (series_cursor*)cur; + *pRowid = pCur->iValue; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int seriesEof(sqlite3_vtab_cursor *cur){ + series_cursor *pCur = (series_cursor*)cur; + return pCur->bDone; +} + +/* True to cause run-time checking of the start=, stop=, and/or step= +** parameters. The only reason to do this is for testing the +** constraint checking logic for virtual tables in the SQLite core. +*/ +#ifndef SQLITE_SERIES_CONSTRAINT_VERIFY +# define SQLITE_SERIES_CONSTRAINT_VERIFY 0 +#endif + +/* +** Return the number of steps between pCur->iBase and pCur->iTerm if +** the step width is pCur->iStep. +*/ +static sqlite3_uint64 seriesSteps(series_cursor *pCur){ + if( pCur->bDesc ){ + assert( pCur->iBase >= pCur->iTerm ); + return span64(pCur->iBase, pCur->iTerm)/pCur->iStep; + }else{ + assert( pCur->iBase <= pCur->iTerm ); + return span64(pCur->iTerm, pCur->iBase)/pCur->iStep; + } +} + +#if defined(SQLITE_ENABLE_MATH_FUNCTIONS) || defined(_WIN32) +/* +** Case 1 (the most common case): +** The standard math library is available so use ceil() and floor() from there. +*/ +static double seriesCeil(double r){ return ceil(r); } +static double seriesFloor(double r){ return floor(r); } +#elif defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +/* +** Case 2 (2nd most common): Use GCC/Clang builtins +*/ +static double seriesCeil(double r){ return __builtin_ceil(r); } +static double seriesFloor(double r){ return __builtin_floor(r); } +#else +/* +** Case 3 (rarely happens): Use home-grown ceil() and floor() routines. +*/ +static double seriesCeil(double r){ + sqlite3_int64 x; + if( r!=r ) return r; + if( r<=(-4503599627370496.0) ) return r; + if( r>=(+4503599627370496.0) ) return r; + x = (sqlite3_int64)r; + if( r==(double)x ) return r; + if( r>(double)x ) x++; + return (double)x; +} +static double seriesFloor(double r){ + sqlite3_int64 x; + if( r!=r ) return r; + if( r<=(-4503599627370496.0) ) return r; + if( r>=(+4503599627370496.0) ) return r; + x = (sqlite3_int64)r; + if( r==(double)x ) return r; + if( r<(double)x ) x--; + return (double)x; +} +#endif + +/* +** This method is called to "rewind" the series_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to seriesColumn() or seriesRowid() or +** seriesEof(). +** +** The query plan selected by seriesBestIndex is passed in the idxNum +** parameter. (idxStr is not used in this implementation.) idxNum +** is a bitmask showing which constraints are available: +** +** 0x0001: start=VALUE +** 0x0002: stop=VALUE +** 0x0004: step=VALUE +** 0x0008: descending order +** 0x0010: ascending order +** 0x0020: LIMIT VALUE +** 0x0040: OFFSET VALUE +** 0x0080: value=VALUE +** 0x0100: value>=VALUE +** 0x0200: value>VALUE +** 0x1000: value<=VALUE +** 0x2000: value0, the value of the LIMIT */ + sqlite3_int64 iOffset = 0; /* if >0, the value of the OFFSET */ + + (void)idxStrUnused; + + /* If any constraints have a NULL value, then return no rows. + ** See ticket https://sqlite.org/src/info/fac496b61722daf2 + */ + for(i=0; i