casadi_os.cpp

/*
 *    This file is part of CasADi.
 *
 *    CasADi -- A symbolic framework for dynamic optimization.
 *    Copyright (C) 2010 by Joel Andersson, Moritz Diehl, K.U.Leuven. All rights reserved.
 *
 *    CasADi is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License as published by the Free Software Foundation; either
 *    version 3 of the License, or (at your option) any later version.
 *
 *    CasADi is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with CasADi; if not, write to the Free Software
 *    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 *
 */
 
#include "casadi_os.hpp"
#include "exception.hpp"
#include "global_options.hpp"
#include <bitset>
#include <cstdlib>
#include <cstring>
#ifdef CASADI_WITH_THREAD
#ifdef CASADI_WITH_THREAD_MINGW
#include <mingw.mutex.h>
#else // CASADI_WITH_THREAD_MINGW
#include <mutex>
#endif // CASADI_WITH_THREAD_MINGW
#endif //CASADI_WITH_THREAD
#ifdef __EMSCRIPTEN__
#include <set>
#endif
 
#ifndef _WIN32
#ifdef WITH_DEEPBIND
#ifndef __APPLE__
#if __GLIBC__
extern char **environ;
#endif
#endif
#endif
#endif
 
 
#ifdef _WIN32
#include <windows.h>
#include <fcntl.h>
#include <io.h>
#endif
 
namespace casadi {
 
// http://stackoverflow.com/questions/303562/c-format-macro-inline-ostringstream
#define STRING(ITEMS) \
  ((dynamic_cast<std::ostringstream &>(std::ostringstream() \
    . seekp(0, std::ios_base::cur) << (ITEMS))) . str())
 
char pathsep() {
    #ifdef _WIN32
        return ';';
    #else
        return ':';
    #endif
}
std::string filesep() {
    #ifdef _WIN32
        return "\\";
    #else
        return "/";
    #endif
}
 
std::vector<std::string> get_search_paths() {
 
    // Build up search paths;
    std::vector<std::string> search_paths;
 
    // Search path: CASADI_PLUGIN_SEARCH_PATH env variable
    // (highest priority; takes precedence over the bundled install dir
    //  that the Python wrapper writes into GlobalOptions::casadipath)
    char* pPLUGIN = getenv("CASADI_PLUGIN_SEARCH_PATH");
    if (pPLUGIN!=nullptr) {
        std::stringstream pluginpaths(pPLUGIN);
        std::string pluginpath;
        while (std::getline(pluginpaths, pluginpath, pathsep())) {
            search_paths.push_back(pluginpath);
        }
    }
 
    // Search path: global casadipath option
    std::stringstream casadipaths(GlobalOptions::getCasadiPath());
    std::string casadipath;
    while (std::getline(casadipaths, casadipath, pathsep())) {
        search_paths.push_back(casadipath);
    }
 
    // Search path: CASADIPATH env variable
    char* pLIBDIR;
    pLIBDIR = getenv("CASADIPATH");
 
    if (pLIBDIR!=nullptr) {
        std::stringstream casadipaths(pLIBDIR);
        std::string casadipath;
        while (std::getline(casadipaths, casadipath, pathsep())) {
        search_paths.push_back(casadipath);
        }
    }
 
    // Search path: bare
    search_paths.push_back("");
 
    // Search path : PLUGIN_EXTRA_SEARCH_PATH
    #ifdef PLUGIN_EXTRA_SEARCH_PATH
    search_paths.push_back(
        std::string("") + PLUGIN_EXTRA_SEARCH_PATH);
    #endif // PLUGIN_EXTRA_SEARCH_PATH
 
    // Search path : current directory
    search_paths.push_back(".");
 
    return search_paths;
}
 
#ifdef _WIN32
// Forward declaration; defined below.
std::wstring utf8_to_utf16(const std::string& s);
#endif
 
#ifdef WITH_DL
 
#ifndef _WIN32
#ifdef WITH_DEEPBIND
#if !defined(__APPLE__) && !defined(__EMSCRIPTEN__)
#if __GLIBC__
namespace {
  // Copy relocation gives a process two distinct `environ` objects: the live one in the
  // executable's .bss, and a permanently-NULL one in glibc's .bss. Code loaded under
  // RTLD_DEEPBIND binds to the latter and therefore sees no environment at all --
  // getenv() is unaffected, but anything iterating environ directly (every OpenMP
  // runtime does, see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111556) silently
  // reads nothing. See also https://github.com/conda-forge/casadi-feedstock/issues/93
  //
  // We publish a snapshot into that dead slot. The snapshot is ours, so glibc can never
  // free it and it never has to be restored -- restoring is what caused casadi#4317
  // (writes NULL back, later readers null-deref) and casadi#4373 (leaves a pointer that
  // glibc's setenv may realloc away).
  //
  // A superseded snapshot can still be held by an earlier plugin, so it is never freed.
  // valgrind reports those as definitely lost; see the casadi/environ_snapshot entry in
  // test/internal/valgrind-casadi.supp.
  char** environ_snapshot = nullptr;
  std::size_t environ_snapshot_n = 0;
#ifdef CASADI_WITH_THREAD
  std::mutex environ_snapshot_mutex;
#endif //CASADI_WITH_THREAD
 
  void publish_environ_snapshot() {
    char*** slot = reinterpret_cast<char***>(dlsym(RTLD_NEXT, "environ"));
    if (!slot || slot == &environ) return;   // no duplicate symbol: nothing to do
#ifdef CASADI_WITH_THREAD
    std::lock_guard<std::mutex> lock(environ_snapshot_mutex);
#endif //CASADI_WITH_THREAD
    std::size_t n = 0;
    if (environ) while (environ[n]) ++n;
    if (environ_snapshot && n == environ_snapshot_n &&
        std::memcmp(environ_snapshot, environ, n * sizeof(char*)) == 0) {
      *slot = environ_snapshot;              // unchanged: republish, allocate nothing
      return;
    }
    char** fresh = static_cast<char**>(std::malloc((n + 1) * sizeof(char*)));
    if (!fresh) return;                      // OOM: leave the slot as it was
    if (n) std::memcpy(fresh, environ, n * sizeof(char*));
    fresh[n] = nullptr;
    environ_snapshot = fresh;                // previous snapshot deliberately leaked:
    environ_snapshot_n = n;                  // an earlier plugin may still hold it
    *slot = environ_snapshot;
  }
}  // namespace
#endif
#endif
#endif
#endif
 
handle_t open_shared_library(const std::string& lib, const std::vector<std::string> &search_paths,
    const std::string& caller, bool global) {
        std::string resultpath;
        return open_shared_library(lib, search_paths, resultpath, caller, global);
}
 
int close_shared_library(handle_t handle) {
    #ifdef _WIN32
        return !FreeLibrary(handle);
    #else // _WIN32
        return dlclose(handle);
    #endif // _WIN32
}
 
handle_t open_shared_library(const std::string& lib, const std::vector<std::string> &search_paths,
        std::string& resultpath, const std::string& caller, bool global) {
    // Alocate a handle
    handle_t handle = nullptr;
 
    // Alocate a handle pointer
    #ifndef _WIN32
        int flag;
        if (global) {
            flag = RTLD_NOW | RTLD_GLOBAL;
        } else {
            flag = RTLD_LAZY | RTLD_LOCAL;
        }
    #ifdef WITH_DEEPBIND
    #if !defined(__APPLE__) && !defined(__EMSCRIPTEN__)
        flag |= RTLD_DEEPBIND;
 
        #if __GLIBC__
        // Hand DEEPBIND-loaded code an environment it can iterate. Never restored;
        // see the comment on publish_environ_snapshot above.
        publish_environ_snapshot();
        #endif
    #endif
    #endif
    #endif
 
 
    // Prepare error string
    std::stringstream errors;
    errors << caller << ": Cannot load shared library '"
           << lib << "': " << std::endl;
    errors << "   (\n"
           << "    Searched directories: 1. CASADI_PLUGIN_SEARCH_PATH env var\n"
           << "                          2. casadipath from GlobalOptions\n"
           << "                          3. CASADIPATH env var\n"
           << "                          4. PATH env var (Windows)\n"
           << "                          5. LD_LIBRARY_PATH env var (Linux)\n"
           << "                          6. DYLD_LIBRARY_PATH env var (osx)\n"
           << "    A library may be 'not found' even if the file exists:\n"
           << "          * library is not ABI-compatible (different compiler/bitness)\n"
           << "          * the dependencies are not found\n"
           << "          * the dependencies are found but have an ABI-incompatible version/compiler/bitness\n" // NOLINT(whitespace/line_length)
           << "   )";
 
    std::string searchpath;
 
#ifdef _WIN32
    // Pass 1 (Windows): strict per-path search.
    //
    // For each non-empty searchpath, do AddDllDirectory + LoadLibraryEx with
    //   LOAD_LIBRARY_SEARCH_USER_DIRS | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS
    //   | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR.
    // PATH and CWD are NOT consulted in this pass. Transitive deps in the
    // same folder as the wrapper resolve via DLL_LOAD_DIR; deps already in
    // the process resolve via the loaded-module list (a pre-filesystem
    // rule). Pass 1 succeeds only when the search dir is self-contained for
    // anything not already loaded -- on failure we fall through to pass 2,
    // which preserves CasADi's legacy semantics (incl. PATH).
    {
        std::wstring libW = utf8_to_utf16(lib);
        for (const std::string& sp : search_paths) {
            if (sp.empty()) continue;
            std::wstring spW = utf8_to_utf16(sp);
            DLL_DIRECTORY_COOKIE cookie = AddDllDirectory(spW.c_str());
            handle = LoadLibraryExW(libW.c_str(), NULL,
                LOAD_LIBRARY_SEARCH_USER_DIRS |
                LOAD_LIBRARY_SEARCH_DEFAULT_DIRS |
                LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
            if (cookie) RemoveDllDirectory(cookie);
            if (handle) {
                resultpath = sp;
                break;
            }
        }
    }
#endif // _WIN32
 
    // Pass 2 (Windows fallback; sole pass on Linux/macOS):
    // Existing legacy loop. On Windows, preserves SetDllDirectory's slot-2
    // hint for transitive deps and the standard search incl. PATH.
    if (!handle) {
#ifdef __EMSCRIPTEN__
    // Emscripten resolves "lib" and "./lib" to the SAME MEMFS module, and a
    // first dlopen that can't complete synchronously (module not resident --
    // e.g. a not-yet-fetched lazy plugin) leaves a poisoned "loading" entry;
    // a second dlopen of the same module then aborts with "...a second time".
    // So attempt each canonical name at most once.  (Native loaders keep the
    // full per-search-path loop below.)
    std::set<std::string> em_tried;
#endif // __EMSCRIPTEN__
    for (casadi_int i=0;i<search_paths.size();++i) {
      searchpath = search_paths[i];
#ifdef _WIN32
      SetDllDirectory(TEXT(searchpath.c_str()));
      handle = LoadLibrary(TEXT(lib.c_str()));
      SetDllDirectory(NULL);
#else // _WIN32
      std::string libname = searchpath.empty() ? lib : searchpath + filesep() + lib;
#ifdef __EMSCRIPTEN__
      if (libname.rfind("./", 0) == 0) libname.erase(0, 2);  // canonicalize
      if (!em_tried.insert(libname).second) continue;        // already tried
#endif // __EMSCRIPTEN__
      handle = dlopen(libname.c_str(), flag);
#endif // _WIN32
      if (handle) {
        resultpath = searchpath;
        break;
      } else {
        errors << std::endl << "  Tried '" << searchpath << "' :";
#ifdef _WIN32
        errors << std::endl << "    Error code (WIN32): " << STRING(GetLastError());
#else // _WIN32
        errors << std::endl << "    Error code: " << dlerror();
#endif // _WIN32
      }
    }
    }
 
    #ifndef _WIN32
    #ifdef WITH_DEEPBIND
    #if !defined(__APPLE__) && !defined(__EMSCRIPTEN__)
    #if __GLIBC__
        // Pick up any setenv the constructors just performed, so code that reads environ
        // lazily (rather than at load time) does not lag a load behind. Allocates nothing
        // unless they changed something.
        publish_environ_snapshot();
    #endif
    #endif
    #endif
    #endif
 
    casadi_assert(handle!=nullptr, errors.str());
 
    return handle;
}
 
#endif // WITH_DL
 
 
// Convert UTF-8 to UTF-16 on Windows
#ifdef _WIN32
std::wstring utf8_to_utf16(const std::string& s) {
    int wlen = MultiByteToWideChar(CP_UTF8, 0, s.data(), static_cast<int>(s.size()), nullptr, 0);
    if (wlen == 0) return {};
    std::wstring ws(wlen, 0);
    MultiByteToWideChar(CP_UTF8, 0, s.data(), static_cast<int>(s.size()), &ws[0], wlen);
    return ws;
}
#endif
 
#ifdef _WIN32
class FdStreamBuf : public std::streambuf {
public:
    explicit FdStreamBuf(int fd, size_t bufsize = 4096)
        : fd_(fd), buffer_(bufsize) {
        setg(buffer_.data(), buffer_.data(), buffer_.data());
    }
 
    ~FdStreamBuf() override {
        if (fd_ >= 0) {
            _close(fd_);
        }
    }
 
protected:
    int_type underflow() override {
        if (gptr() < egptr()) {
            return traits_type::to_int_type(*gptr());
        }
 
        int n = _read(fd_, buffer_.data(), static_cast<unsigned int>(buffer_.size()));
        if (n <= 0) {
            return traits_type::eof();
        }
 
        setg(buffer_.data(), buffer_.data(), buffer_.data() + n);
        return traits_type::to_int_type(*gptr());
    }
 
    std::streampos seekoff(std::streamoff off, std::ios_base::seekdir dir,
                            std::ios_base::openmode which = std::ios_base::in) override {
        if (!(which & std::ios_base::in)) return -1;
 
        __int64 whence;
        switch (dir) {
            case std::ios_base::beg: whence = SEEK_SET; break;
            case std::ios_base::cur:
                // Need to include the offset in buffer
                off -= egptr() - gptr();
                whence = SEEK_CUR;
                break;
            case std::ios_base::end: whence = SEEK_END; break;
            default: return -1;
        }
 
        __int64 result = _lseeki64(fd_, off, static_cast<int>(whence));
        if (result == -1) {
            return -1;
        }
 
        // Invalidate the buffer
        setg(buffer_.data(), buffer_.data(), buffer_.data());
        return result;
    }
 
    std::streampos seekpos(std::streampos pos,
                            std::ios_base::openmode which = std::ios_base::in) override {
        return seekoff(static_cast<std::streamoff>(pos), std::ios_base::beg, which);
    }
 
 
private:
    int fd_;
    std::vector<char> buffer_;
};
 
class FdOStreamBuf : public std::streambuf {
public:
    explicit FdOStreamBuf(int fd, size_t bufsize = 4096)
        : fd_(fd), buffer_(bufsize) {
        setp(buffer_.data(), buffer_.data() + buffer_.size());
    }
 
    ~FdOStreamBuf() override {
        sync(); // flush on destruction
        if (fd_ >= 0) {
            _close(fd_);
        }
    }
 
protected:
    int_type overflow(int_type ch) override {
        if (flush_buffer() == -1) return traits_type::eof();
 
        if (ch != traits_type::eof()) {
            *pptr() = static_cast<char>(ch);
            pbump(1);
        }
 
        return ch;
    }
 
    int sync() override {
        return flush_buffer() == -1 ? -1 : 0;
    }
 
private:
    int flush_buffer() {
        int len = static_cast<int>(pptr() - pbase());
        if (len > 0) {
            int written = _write(fd_, pbase(), len);
            if (written != len) return -1;
            pbump(-len);
        }
        return 0;
    }
 
    int fd_;
    std::vector<char> buffer_;
};
 
struct OwnedIStream {
    std::unique_ptr<FdStreamBuf> buffer;
    std::unique_ptr<std::istream> stream;
 
    explicit OwnedIStream(int fd)
        : buffer(std::make_unique<FdStreamBuf>(fd)),
          stream(std::make_unique<std::istream>(buffer.get())) {}
};
 
struct StreamWithOwnedBuffer : public std::istream {
    std::shared_ptr<OwnedIStream> owned;
    explicit StreamWithOwnedBuffer(std::shared_ptr<OwnedIStream> o)
        : std::istream(o->buffer.get()), owned(std::move(o)) {}
};
 
struct OwnedOStream {
    std::unique_ptr<FdOStreamBuf> buffer;
    std::unique_ptr<std::ostream> stream;
 
    explicit OwnedOStream(int fd)
        : buffer(std::make_unique<FdOStreamBuf>(fd)),
          stream(std::make_unique<std::ostream>(buffer.get())) {}
};
 
struct StreamWithOwnedOBuffer : public std::ostream {
    std::shared_ptr<OwnedOStream> owned;
    explicit StreamWithOwnedOBuffer(std::shared_ptr<OwnedOStream> o)
        : std::ostream(o->buffer.get()), owned(std::move(o)) {}
};
#endif
 
// Portable ifstream opener that supports UTF-8 filenames on Windows
std::unique_ptr<std::istream> ifstream_compat(const std::string& utf8_path,
                                    std::ios::openmode mode) {
#ifdef _WIN32
    std::wstring utf16_path = utf8_to_utf16(utf8_path);
    DWORD access = 0;
    access |= GENERIC_READ;
    if (mode & std::ios::out) access |= GENERIC_WRITE;
 
    HANDLE h = CreateFileW(utf16_path.c_str(), access,
                             FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr,
                           OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
    if (h == INVALID_HANDLE_VALUE) return {};
 
    int flags = (mode & std::ios::out) ? _O_RDWR : _O_RDONLY;
    if (mode & std::ios::binary) flags |= _O_BINARY;
 
    int fd = _open_osfhandle(reinterpret_cast<intptr_t>(h), flags);
    if (fd == -1) {
        CloseHandle(h);
        return {};
    }
    auto owned = std::make_shared<OwnedIStream>(fd);
    return std::unique_ptr<StreamWithOwnedBuffer>(new StreamWithOwnedBuffer(std::move(owned)));
#else
    auto ifs = std::unique_ptr<std::ifstream>(new std::ifstream(utf8_path, mode));
    if (!*ifs) return {};
    return std::unique_ptr<std::istream>(std::move(ifs));
#endif
}
 
std::unique_ptr<std::ostream> ofstream_compat(const std::string& utf8_path,
                                              std::ios::openmode mode) {
#ifdef _WIN32
    std::wstring utf16_path = utf8_to_utf16(utf8_path);
 
    DWORD access = 0;
    access |= GENERIC_WRITE;
    if (mode & std::ios::in)   access |= GENERIC_READ;
 
    DWORD creation = (mode & std::ios::app) ? OPEN_ALWAYS : CREATE_ALWAYS;
 
    HANDLE h = CreateFileW(utf16_path.c_str(), access,
                            FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr,
                           creation, FILE_ATTRIBUTE_NORMAL, nullptr);
    if (h == INVALID_HANDLE_VALUE) return {};
 
    int flags = (mode & std::ios::in) ? _O_RDWR : _O_WRONLY;
    if (mode & std::ios::app) flags |= _O_APPEND;
    if (mode & std::ios::binary) {
        flags |= _O_BINARY;
    } else {
        flags |= _O_TEXT;
    }
 
    int fd = _open_osfhandle(reinterpret_cast<intptr_t>(h), flags);
    if (fd == -1) {
        CloseHandle(h);
        return {};
    }
 
    auto owned = std::make_shared<OwnedOStream>(fd);
    return std::unique_ptr<StreamWithOwnedOBuffer>(new StreamWithOwnedOBuffer(std::move(owned)));
#else
    auto ofs = std::unique_ptr<std::ofstream>(new std::ofstream(utf8_path, mode));
    if (!*ofs) return {};
    return std::unique_ptr<std::ostream>(std::move(ofs));
#endif
}
 
} // namespace casadi