From 392096536cea6bbad0e8de8b6f7ecf06d07124bd Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Fri, 29 Dec 2023 10:47:29 +0100 Subject: [PATCH 1/7] MDEV-30432: Remove cpprestsdk from Connect - Remove GetRest occurance - Add libcurl feature to ConnectSE - This patch closes MDEV-26727 (tested with Docker) --- storage/connect/CMakeLists.txt | 38 +- .../connect/mysql-test/connect/t/rest.test | 3 +- storage/connect/plgdbsem.h | 1 - storage/connect/restget.cpp | 90 ---- storage/connect/tabrest.cpp | 400 +++++++----------- storage/connect/tabrest.h | 59 ++- 6 files changed, 214 insertions(+), 377 deletions(-) delete mode 100644 storage/connect/restget.cpp diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 65fb6fbf19b38..64f79c7828dd7 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -360,31 +360,21 @@ ENDIF(CONNECT_WITH_MONGO) OPTION(CONNECT_WITH_REST "Compile CONNECT storage engine with REST support" ON) IF(CONNECT_WITH_REST) -# MESSAGE(STATUS "=====> REST support is ON") - SET(CONNECT_SOURCES ${CONNECT_SOURCES} tabrest.cpp tabrest.h) - add_definitions(-DREST_SUPPORT) -# FIND_PACKAGE(cpprestsdk QUIET) -# IF (cpprestsdk_FOUND) -# IF(UNIX) -## INCLUDE_DIRECTORIES(${CPPRESTSDK_INCLUDE_DIR}) -## If needed edit next line to set the path to libcpprest.so -# SET(REST_LIBRARY -lcpprest) -# MESSAGE (STATUS ${REST_LIBRARY}) -# ELSE(NOT UNIX) -## Next line sets debug compile mode matching cpprest_2_10d.dll -## when it was binary installed (can be change later in Visual Studio) -## Comment it out if not needed depending on your cpprestsdk installation. -# SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MDd") -# ENDIF(UNIX) -## IF(REST_LIBRARY) why this? how about Windows -# SET(CONNECT_SOURCES ${CONNECT_SOURCES} restget.cpp) -# add_definitions(-DREST_SOURCE) -## ENDIF() -## ELSE(NOT cpprestsdk_FOUND) -# MESSAGE(STATUS "=====> cpprestsdk package not found") -# ENDIF (cpprestsdk_FOUND) + SET(REST_LIBRARY) + find_package(CURL) + SET_PACKAGE_PROPERTIES(Curl PROPERTIES TYPE REQUIRED + PURPOSE "Required for the CONNECT_WITH_REST feature") + IF (CURL_FOUND) + SET(REST_LIBRARY ${CURL_LIBRARIES}) + MESSAGE (STATUS ${REST_LIBRARY}) + SET(CONNECT_SOURCES ${CONNECT_SOURCES} tabrest.cpp tabrest.h) + add_definitions(-DREST_SUPPORT) + ADD_FEATURE_INFO(CONNECT_REST "ON" "Support for REST API in the CONNECT storage engine") + ELSE() + MESSAGE_ONCE(CONNECT_NO_CURL "libcurl-dev header not found.") + ADD_FEATURE_INFO(CONNECT_REST "OFF" "Support for REST API in the CONNECT storage engine") + ENDIF() ENDIF(CONNECT_WITH_REST) -ADD_FEATURE_INFO(CONNECT_REST CONNECT_WITH_REST "Support for REST API in the CONNECT storage engine") # # XMAP diff --git a/storage/connect/mysql-test/connect/t/rest.test b/storage/connect/mysql-test/connect/t/rest.test index 67066ed4639e9..200f617492fcb 100644 --- a/storage/connect/mysql-test/connect/t/rest.test +++ b/storage/connect/mysql-test/connect/t/rest.test @@ -12,6 +12,5 @@ SELECT * FROM t1; DROP TABLE t1; # -# Clean up +# Clean up is done automatically # ---remove_file $MYSQLD_DATADIR/test/users.json diff --git a/storage/connect/plgdbsem.h b/storage/connect/plgdbsem.h index 11a763fa941b9..72237f853d1da 100644 --- a/storage/connect/plgdbsem.h +++ b/storage/connect/plgdbsem.h @@ -404,7 +404,6 @@ typedef class VCTDEF *PVCTDEF; typedef class PIVOTDEF *PPIVOTDEF; typedef class DOMDEF *PDOMDEF; typedef class DIRDEF *PDIRDEF; -typedef class RESTDEF *PRESTDEF; typedef class OEMDEF *POEMDEF; typedef class COLCRT *PCOLCRT; typedef class COLDEF *PCOLDEF; diff --git a/storage/connect/restget.cpp b/storage/connect/restget.cpp deleted file mode 100644 index 29dae23078066..0000000000000 --- a/storage/connect/restget.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/************* Restget C++ Program Source Code File (.CPP) *************/ -/* Adapted from the sample program of the Casablanca tutorial. */ -/* Copyright Olivier Bertrand 2019. */ -/***********************************************************************/ -#include -#include - -using namespace utility::conversions; // String conversions utilities -using namespace web; // Common features like URIs. -using namespace web::http; // Common HTTP functionality -using namespace web::http::client; // HTTP client features -using namespace concurrency::streams; // Asynchronous streams - -typedef const char* PCSZ; - -extern "C" int restGetFile(char* m, bool xt, PCSZ http, PCSZ uri, PCSZ fn); - -/***********************************************************************/ -/* Make a local copy of the requested file. */ -/***********************************************************************/ -int restGetFile(char *m, bool xt, PCSZ http, PCSZ uri, PCSZ fn) -{ - int rc = 0; - auto fileStream = std::make_shared(); - - if (!http || !fn) { - //strcpy(g->Message, "Missing http or filename"); - strcpy(m, "Missing http or filename"); - return 2; - } // endif - - if (xt) - fprintf(stderr, "restGetFile: fn=%s\n", fn); - - // Open stream to output file. - pplx::task requestTask = fstream::open_ostream(to_string_t(fn)) - .then([=](ostream outFile) { - *fileStream= outFile; - - if (xt) - fprintf(stderr, "Outfile isopen=%d\n", outFile.is_open()); - - // Create http_client to send the request. - http_client client(to_string_t(http)); - - if (uri) { - // Build request URI and start the request. - uri_builder builder(to_string_t(uri)); - return client.request(methods::GET, builder.to_string()); - } else - return client.request(methods::GET); - }) - - // Handle response headers arriving. - .then([=](http_response response) { - if (xt) - fprintf(stderr, "Received response status code:%u\n", - response.status_code()); - - // Write response body into the file. - return response.body().read_to_end(fileStream->streambuf()); - }) - - // Close the file stream. - .then([=](size_t n) { - if (xt) - fprintf(stderr, "Return size=%zu\n", n); - - return fileStream->close(); - }); - - // Wait for all the outstanding I/O to complete and handle any exceptions - try { - if (xt) - fprintf(stderr, "Waiting\n"); - - requestTask.wait(); - } catch (const std::exception &e) { - if (xt) - fprintf(stderr, "Error exception: %s\n", e.what()); - - sprintf(m, "Error exception: %s", e.what()); - rc= 1; - } // end try/catch - - if (xt) - fprintf(stderr, "restget done: rc=%d\n", rc); - - return rc; -} // end of restGetFile diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index cb8135d7d723e..27e75161eea72 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -36,6 +36,7 @@ #include "tabjson.h" #include "tabfmt.h" #include "tabrest.h" +#include #if defined(connect_EXPORTS) #define PUSH_WARNING(M) push_warning(current_thd, Sql_condition::WARN_LEVEL_NOTE, ER_UNKNOWN_ERROR, M) @@ -43,234 +44,69 @@ #define PUSH_WARNING(M) htrc(M) #endif -static XGETREST getRestFnc = NULL; -static int Xcurl(PGLOBAL g, PCSZ Http, PCSZ Uri, PCSZ filename); - -/***********************************************************************/ -/* Xcurl: retrieve the REST answer by executing cURL. */ -/***********************************************************************/ -int Xcurl(PGLOBAL g, PCSZ Http, PCSZ Uri, PCSZ filename) -{ - char buf[512]; - int rc = 0; - - if (strchr(filename, '"')) { - strcpy(g->Message, "Invalid file name"); - return 1; - } // endif filename - - if (Uri) { - if (*Uri == '/' || Http[strlen(Http) - 1] == '/') - my_snprintf(buf, sizeof(buf)-1, "%s%s", Http, Uri); - else - my_snprintf(buf, sizeof(buf)-1, "%s/%s", Http, Uri); - - } else - my_snprintf(buf, sizeof(buf)-1, "%s", Http); - -#if defined(_WIN32) - char cmd[1024]; - STARTUPINFO si; - PROCESS_INFORMATION pi; - - sprintf(cmd, "curl \"%s\" -o \"%s\"", buf, filename); - - ZeroMemory(&si, sizeof(si)); - si.cb = sizeof(si); - ZeroMemory(&pi, sizeof(pi)); - - // Start the child process. - if (CreateProcess(NULL, cmd, NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi)) { - // Wait until child process exits. - WaitForSingleObject(pi.hProcess, INFINITE); - - // Close process and thread handles. - CloseHandle(pi.hProcess); - CloseHandle(pi.hThread); - } else { - snprintf(g->Message, sizeof(g->Message), "CreateProcess curl failed (%d)", GetLastError()); - rc = 1; - } // endif CreateProcess -#else // !_WIN32 - char fn[600]; - pid_t pID; - - // Check if curl package is available by executing subprocess - FILE *f= popen("command -v curl", "r"); - - if (!f) { - strcpy(g->Message, "Problem in allocating memory."); - return 1; - } else { - char temp_buff[50]; - size_t len = fread(temp_buff,1, 50, f); - - if(!len) { - strcpy(g->Message, "Curl not installed."); - return 1; - } else - pclose(f); - - } // endif f - -#ifdef HAVE_VFORK - pID = vfork(); -#else - pID = fork(); -#endif - sprintf(fn, "-o%s", filename); - - if (pID == 0) { - // Code executed by child process - execlp("curl", "curl", buf, fn, (char*)NULL); - - // If execlp() is successful, we should not reach this next line. - strcpy(g->Message, "Unsuccessful execlp from vfork()"); - exit(1); - } else if (pID < 0) { - // failed to fork - strcpy(g->Message, "Failed to fork"); - rc = 1; - } else { - // Parent process - wait(NULL); // Wait for the child to terminate - } // endif pID -#endif // !_WIN32 - - return rc; -} // end of Xcurl - -/***********************************************************************/ -/* GetREST: load the Rest lib and get the Rest function. */ -/***********************************************************************/ -XGETREST GetRestFunction(PGLOBAL g) -{ - if (getRestFnc) - return getRestFnc; - -#if !defined(REST_SOURCE) - if (trace(515)) - htrc("Looking for GetRest library\n"); - -#if defined(_WIN32) || defined(_WINDOWS) - HANDLE Hdll; - const char* soname = "GetRest.dll"; // Module name - - if (!(Hdll = LoadLibrary(soname))) { - char buf[256]; - DWORD rc = GetLastError(); - - snprintf(g->Message, sizeof(g->Message), MSG(DLL_LOAD_ERROR), rc, soname); - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0, - (LPTSTR)buf, sizeof(buf), NULL); - strcat(strcat(g->Message, ": "), buf); - return NULL; - } // endif Hdll - -// Get the function returning an instance of the external DEF class - if (!(getRestFnc = (XGETREST)GetProcAddress((HINSTANCE)Hdll, "restGetFile"))) { - char buf[256]; - DWORD rc = GetLastError(); - - snprintf(g->Message, sizeof(g->Message), MSG(PROCADD_ERROR), rc, "restGetFile"); - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0, - (LPTSTR)buf, sizeof(buf), NULL); - strcat(strcat(g->Message, ": "), buf); - FreeLibrary((HMODULE)Hdll); - return NULL; - } // endif getRestFnc -#else // !_WIN32 - void* Hso; - const char* error = NULL; - const char* soname = "GetRest.so"; // Module name - - // Load the desired shared library - if (!(Hso = dlopen(soname, RTLD_LAZY))) { - error = dlerror(); - snprintf(g->Message, sizeof(g->Message), MSG(SHARED_LIB_ERR), soname, SVP(error)); - return NULL; - } // endif Hdll - -// Get the function returning an instance of the external DEF class - if (!(getRestFnc = (XGETREST)dlsym(Hso, "restGetFile"))) { - error = dlerror(); - snprintf(g->Message, sizeof(g->Message), MSG(GET_FUNC_ERR), "restGetFile", SVP(error)); - dlclose(Hso); - return NULL; - } // endif getdef -#endif // !_WIN32 -#else // REST_SOURCE - getRestFnc = restGetFile; -#endif // REST_SOURCE - - return getRestFnc; -} // end of GetRestFunction - /***********************************************************************/ /* Return the columns definition to MariaDB. */ /***********************************************************************/ PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) { PQRYRES qrp= NULL; + RESTDEF restObject; char filename[_MAX_PATH + 1]; // MAX PATH ??? - int rc; + int rc; PCSZ http, uri, fn, ftype; - XGETREST grf = NULL; - bool curl = GetBooleanTableOption(g, tp, "Curl", false); - if (!curl && !(grf = GetRestFunction(g))) - curl = true; http = GetStringTableOption(g, tp, "Http", NULL); uri = GetStringTableOption(g, tp, "Uri", NULL); ftype = GetStringTableOption(g, tp, "Type", "JSON"); - fn = GetStringTableOption(g, tp, "Filename", NULL); - - if (!fn) { - int n, m = strlen(ftype) + 1; - - strcat(strcpy(filename, tab), "."); - n = strlen(filename); - - // Fold ftype to lower case - for (int i = 0; i < m; i++) - filename[n + i] = tolower(ftype[i]); - - fn = filename; - tp->subtype = PlugDup(g, fn); - snprintf(g->Message, sizeof(g->Message), "No file name. Table will use %s", fn); - PUSH_WARNING(g->Message); - } // endif fn + fn = GetStringTableOption(g, tp, "Filename", NULL); + + if (!fn) + { + int n, m = strlen(ftype) + 1; + strcat(strcpy(filename, tab), "."); + n = strlen(filename); + // Fold ftype to lower case + for (int i = 0; i < m; i++) + filename[n + i] = tolower(ftype[i]); + fn = filename; + tp->subtype = PlugDup(g, fn); + snprintf(g->Message, sizeof(g->Message), "No file name. Table will use %s", fn); + PUSH_WARNING(g->Message); + } // We used the file name relative to recorded datapath - PlugSetPath(filename, fn, db); - remove(filename); - - // Retrieve the file from the web and copy it locally - if (curl) - rc = Xcurl(g, http, uri, filename); - else - rc = grf(g->Message, trace(515), http, uri, filename); - - if (rc) { - strcpy(g->Message, "Cannot access to curl nor casablanca"); - return NULL; - } else if (!stricmp(ftype, "JSON")) + PlugSetPath(filename, fn, db); + restObject.Http= http; + restObject.Uri= uri; + restObject.Fn= filename; + remove(filename); + // Retrieve the file from the web using curl and copy it locally + if (restObject.curl_init(g)) + { + snprintf(g->Message, sizeof(g->Message), "Initialization of curl failed."); + return NULL; + } + rc = restObject.curl_run(g); + if (rc) + { + snprintf(g->Message, sizeof(g->Message), "Cannot access to curl."); + return NULL; + } + else if (!stricmp(ftype, "JSON")) qrp = JSONColumns(g, db, NULL, tp, info); else if (!stricmp(ftype, "CSV")) qrp = CSVColumns(g, NULL, tp, info); #if defined(XML_SUPPORT) - else if (!stricmp(ftype, "XML")) - qrp = XMLColumns(g, db, tab, tp, info); + else if (!stricmp(ftype, "XML")) + qrp = XMLColumns(g, db, tab, tp, info); #endif // XML_SUPPORT - else + else snprintf(g->Message, sizeof(g->Message), "Usupported file type %s", ftype); - return qrp; } // end of RESTColumns + /* -------------------------- Class RESTDEF -------------------------- */ /***********************************************************************/ @@ -278,18 +114,12 @@ PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) /***********************************************************************/ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) { - char filename[_MAX_PATH + 1]; + char filename[_MAX_PATH + 1]; int rc = 0, n; - bool xt = trace(515); - LPCSTR ftype; - XGETREST grf = NULL; - bool curl = GetBoolCatInfo("Curl", false); - - if (!curl && !(grf = GetRestFunction(g))) - curl = true; + bool xt = trace(515); + LPCSTR ftype; ftype = GetStringCatInfo(g, "Type", "JSON"); - if (xt) htrc("ftype = %s am = %s\n", ftype, SVP(am)); @@ -299,9 +129,11 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) #endif // XML_SUPPORT : (!stricmp(ftype, "CSV")) ? 3 : 0; - if (n == 0) { + if (n == 0) + { htrc("DefineAM: Unsupported REST table type %s\n", ftype); - snprintf(g->Message, sizeof(g->Message), "Unsupported REST table type %s", ftype); + snprintf(g->Message, sizeof(g->Message), + "Unsupported REST table type %s", ftype); return true; } // endif n @@ -311,27 +143,26 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) // We used the file name relative to recorded datapath PlugSetPath(filename, Fn, GetPath()); - remove(filename); - - // Retrieve the file from the web and copy it locally - if (curl) { - rc = Xcurl(g, Http, Uri, filename); - xtrc(515, "Return from Xcurl: rc=%d\n", rc); - } else { - rc = grf(g->Message, xt, Http, Uri, filename); - xtrc(515, "Return from restGetFile: rc=%d\n", rc); - } // endelse - - if (rc) { - // strcpy(g->Message, "Cannot access to curl nor casablanca"); - return true; - } else switch (n) { - case 1: Tdp = new (g) JSONDEF; break; + Fn= filename; + remove(filename); + if (curl_init(g)) + { + snprintf(g->Message, sizeof(g->Message), "Initialization of curl failed."); + return true; + } + if (curl_run(g)) + return true; + else switch (n) + { + case 1: + Tdp = new (g) JSONDEF; break; #if defined(XML_SUPPORT) - case 2: Tdp = new (g) XMLDEF; break; + case 2: + Tdp = new (g) XMLDEF; break; #endif // XML_SUPPORT - case 3: Tdp = new (g) CSVDEF; break; - default: Tdp = NULL; + case 3: + Tdp = new (g) CSVDEF; break; + default: Tdp = NULL; } // endswitch n // Do make the table/view definition @@ -345,6 +176,7 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) return (Tdp == NULL); } // end of DefineAM + /***********************************************************************/ /* GetTable: makes a new Table Description Block. */ /***********************************************************************/ @@ -353,12 +185,106 @@ PTDB RESTDEF::GetTable(PGLOBAL g, MODE m) if (trace(515)) htrc("REST GetTable mode=%d\n", m); - if (m != MODE_READ && m != MODE_READX && m != MODE_ANY) { - strcpy(g->Message, "REST tables are currently read only"); + if (m != MODE_READ && m != MODE_READX && m != MODE_ANY) + { + snprintf(g->Message, sizeof(g->Message), "REST tables are currently read only"); return NULL; - } // endif m + } return Tdp->GetTable(g, m); // Leave file type do the job } // end of GetTable + +/***********************************************************************/ +/* curl_init: Initilize curl */ +/***********************************************************************/ +int RESTDEF::curl_init(PGLOBAL g) +{ + CURLcode curl_res = curl_global_init(CURL_GLOBAL_ALL); + if (curl_res != CURLE_OK) + { + snprintf(g->Message, sizeof(g->Message), + "unable to initialize curl library, " + "curl returned this error code: %u " + "with the following error message: %s", + curl_res, curl_easy_strerror(curl_res)); + return 1; + } + curl_inited = true; + return 0; +} + + +/***********************************************************************/ +/* curl_deinit: Cleanup curl */ +/***********************************************************************/ +void RESTDEF::curl_deinit() +{ + if (curl_inited) + { + curl_global_cleanup(); + curl_inited = false; + } +} + + +/***********************************************************************/ +/* curl_run: Retrieve the REST answer by executing cURL. */ +/***********************************************************************/ +int RESTDEF::curl_run(PGLOBAL g) +{ + CURL *curl = curl_easy_init(); + CURLcode curl_res = CURLE_OK; + char buf[512]; + long http_code = 0; + char curl_errbuf[CURL_ERROR_SIZE]; + if (!curl) + { + snprintf(g->Message, sizeof(g->Message), "Cannot initilize curl session."); + return 1; + } + curl_errbuf[0] = '\0'; + if (Uri) + { + if (*Uri == '/' || Http[strlen(Http) - 1] == '/') + my_snprintf(buf, sizeof(buf)-1, "%s%s", Http, Uri); + else + my_snprintf(buf, sizeof(buf)-1, "%s/%s", Http, Uri); + } + else + my_snprintf(buf, sizeof(buf)-1, "%s", Http); + + FILE *f= fopen(Fn, "wb"); + if ((curl_res= curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errbuf)) != + CURLE_OK || + (curl_res= curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, + NULL)) != + CURLE_OK || + (curl_res= curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) f)) != + CURLE_OK || + (curl_res = curl_easy_setopt(curl, CURLOPT_URL, buf)) != CURLE_OK || + (curl_res = curl_easy_perform(curl)) != CURLE_OK || + (curl_res = curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, + &http_code)) != CURLE_OK) + { + curl_easy_cleanup(curl); + if (curl_res) + { + snprintf(g->Message, sizeof(g->Message), + "curl returned this error code: %u " + "with the following error message: %s", curl_res, + curl_errbuf[0] ? curl_errbuf : curl_easy_strerror(curl_res)); + return 1; + } + } + curl_easy_cleanup(curl); + fclose(f); + bool is_error = http_code < 200 || http_code >= 300; + if (is_error) + { + snprintf(g->Message, sizeof(g->Message), "server error"); + return 1; + } + return 0; +} /* ---------------------- End of Class RESTDEF ----------------------- */ diff --git a/storage/connect/tabrest.h b/storage/connect/tabrest.h index a04117ca82df6..31283e006ff2b 100644 --- a/storage/connect/tabrest.h +++ b/storage/connect/tabrest.h @@ -10,39 +10,52 @@ #define stricmp strcasecmp #endif // !_WIN32 -typedef int(__stdcall* XGETREST) (char*, bool, PCSZ, PCSZ, PCSZ); - /***********************************************************************/ /* Functions used by REST. */ /***********************************************************************/ -XGETREST GetRestFunction(PGLOBAL g); -#if defined(REST_SOURCE) -extern "C" int restGetFile(char* m, bool xt, PCSZ http, PCSZ uri, PCSZ fn); -#endif // REST_SOURCE #if defined(MARIADB) PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char* tab, char* db, bool info); #endif // !MARIADB /***********************************************************************/ -/* Restest table. */ +/* Data structure for curl callback function */ /***********************************************************************/ -class RESTDEF : public TABDEF { /* Table description */ -public: - // Constructor - RESTDEF(void) { Tdp = NULL; Http = Uri = Fn = NULL; } - - // Implementation - const char *GetType(void) override { return "REST"; } +struct MemoryStruct { + char *memory; + size_t size; +}; - // Methods - bool DefineAM(PGLOBAL g, LPCSTR am, int poff) override; - PTDB GetTable(PGLOBAL g, MODE m) override; -protected: - // Members - PRELDEF Tdp; - PCSZ Http; /* Web connection HTTP */ - PCSZ Uri; /* Web connection URI */ - PCSZ Fn; /* The intermediate file name */ +/***********************************************************************/ +/* Restest table. */ +/***********************************************************************/ +class RESTDEF : public TABDEF { /* Table description */ +private: + bool curl_inited; +public: +// Constructor + RESTDEF() + :curl_inited(false), + Tdp(NULL), + Http(NULL), + Uri(NULL), + Fn(NULL) + {} + int curl_init (PGLOBAL g); + void curl_deinit (); + // Methods + const char *GetType(void) override { return "REST"; } + virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff) override; + virtual PTDB GetTable(PGLOBAL g, MODE m) override; + int curl_run(PGLOBAL g); + // Members + PRELDEF Tdp; + PCSZ Http; /* Web connection HTTP */ + PCSZ Uri; /* Web connection URI */ + PCSZ Fn; /* The intermediate file name */ + ~RESTDEF() + { + curl_deinit(); + } }; // end of class RESTDEF From 4a6d99cb82a7e0818b2710aa6c8162cce3ee5276 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 13 May 2026 17:00:18 +1000 Subject: [PATCH 2/7] Revert "MDEV-30509: mariadb-plugin-connect: introduce curl as recommends" This reverts commit 2a78c3ef6fd6663d6731dd5cec2f462420b61123. --- debian/control | 2 -- 1 file changed, 2 deletions(-) diff --git a/debian/control b/debian/control index 0c1df4fac5ab3..b07884be71d36 100644 --- a/debian/control +++ b/debian/control @@ -1034,8 +1034,6 @@ Depends: unixodbc, ${misc:Depends}, ${shlibs:Depends}, -Recommends: - curl, Breaks: mariadb-connect-engine-10.0, mariadb-connect-engine-10.1, From 47c27b32dc955283ff86de3c97019b5b3852448b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 13 May 2026 17:39:16 +1000 Subject: [PATCH 3/7] fix: curl include path for connect-rest --- storage/connect/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 64f79c7828dd7..7398d26538106 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -366,6 +366,7 @@ IF(CONNECT_WITH_REST) PURPOSE "Required for the CONNECT_WITH_REST feature") IF (CURL_FOUND) SET(REST_LIBRARY ${CURL_LIBRARIES}) + INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIRS}) MESSAGE (STATUS ${REST_LIBRARY}) SET(CONNECT_SOURCES ${CONNECT_SOURCES} tabrest.cpp tabrest.h) add_definitions(-DREST_SUPPORT) From 19741da65ee3fcec31d4a3b9dd055da95d66f5c2 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 14 May 2026 16:58:41 +1000 Subject: [PATCH 4/7] connect - filename in object not stack --- storage/connect/tabrest.cpp | 17 +++++++---------- storage/connect/tabrest.h | 5 ++--- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index 27e75161eea72..5364280af4885 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -51,7 +51,6 @@ PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) { PQRYRES qrp= NULL; RESTDEF restObject; - char filename[_MAX_PATH + 1]; // MAX PATH ??? int rc; PCSZ http, uri, fn, ftype; @@ -64,23 +63,22 @@ PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) if (!fn) { int n, m = strlen(ftype) + 1; - strcat(strcpy(filename, tab), "."); - n = strlen(filename); + strcat(strcpy(restObject.filename, tab), "."); + n = strlen(restObject.filename); // Fold ftype to lower case for (int i = 0; i < m; i++) - filename[n + i] = tolower(ftype[i]); - fn = filename; + restObject.filename[n + i] = tolower(ftype[i]); + fn = restObject.filename; tp->subtype = PlugDup(g, fn); snprintf(g->Message, sizeof(g->Message), "No file name. Table will use %s", fn); PUSH_WARNING(g->Message); } // We used the file name relative to recorded datapath - PlugSetPath(filename, fn, db); + PlugSetPath(restObject.filename, fn, db); restObject.Http= http; restObject.Uri= uri; - restObject.Fn= filename; - remove(filename); + remove(restObject.filename); // Retrieve the file from the web using curl and copy it locally if (restObject.curl_init(g)) { @@ -114,10 +112,10 @@ PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) /***********************************************************************/ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) { - char filename[_MAX_PATH + 1]; int rc = 0, n; bool xt = trace(515); LPCSTR ftype; + char *Fn; ftype = GetStringCatInfo(g, "Type", "JSON"); if (xt) @@ -143,7 +141,6 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) // We used the file name relative to recorded datapath PlugSetPath(filename, Fn, GetPath()); - Fn= filename; remove(filename); if (curl_init(g)) { diff --git a/storage/connect/tabrest.h b/storage/connect/tabrest.h index 31283e006ff2b..970916ef76763 100644 --- a/storage/connect/tabrest.h +++ b/storage/connect/tabrest.h @@ -39,8 +39,7 @@ class RESTDEF : public TABDEF { /* Table description */ :curl_inited(false), Tdp(NULL), Http(NULL), - Uri(NULL), - Fn(NULL) + Uri(NULL) {} int curl_init (PGLOBAL g); void curl_deinit (); @@ -53,7 +52,7 @@ class RESTDEF : public TABDEF { /* Table description */ PRELDEF Tdp; PCSZ Http; /* Web connection HTTP */ PCSZ Uri; /* Web connection URI */ - PCSZ Fn; /* The intermediate file name */ + char filename[_MAX_PATH + 1];/* The intermediate file name */ ~RESTDEF() { curl_deinit(); From 1fc67ce6bb1462caf18d0a1620e0070377ee5d31 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 14 May 2026 16:59:23 +1000 Subject: [PATCH 5/7] curl_run: dynamic memory and better error handling --- storage/connect/tabrest.cpp | 43 +++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index 5364280af4885..fd54de9db8115 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -230,28 +230,54 @@ void RESTDEF::curl_deinit() /***********************************************************************/ int RESTDEF::curl_run(PGLOBAL g) { - CURL *curl = curl_easy_init(); + CURL *curl; CURLcode curl_res = CURLE_OK; - char buf[512]; + char *buf; long http_code = 0; char curl_errbuf[CURL_ERROR_SIZE]; + size_t bufsz; + + if (!Http || !*Http) + { + snprintf(g->Message, sizeof(g->Message), "HTTP URL is missing or empty."); + return 1; + } + + FILE *f = fopen(filename, "wb"); + if (!f) + { + snprintf(g->Message, sizeof(g->Message), "Cannot open file %s for writing.", filename); + return 1; + } + curl = curl_easy_init(); if (!curl) { snprintf(g->Message, sizeof(g->Message), "Cannot initilize curl session."); + fclose(f); return 1; } + curl_errbuf[0] = '\0'; + bufsz= Uri ? strlen(Uri) : 0 + strlen(Http) + 2; + buf= (char *) malloc(bufsz); + bufsz--; + if (!buf) + { + snprintf(g->Message, sizeof(g->Message), "Cannot allocate memory for curl url."); + curl_easy_cleanup(curl); + fclose(f); + return 1; + } if (Uri) { if (*Uri == '/' || Http[strlen(Http) - 1] == '/') - my_snprintf(buf, sizeof(buf)-1, "%s%s", Http, Uri); + my_snprintf(buf, bufsz, "%s%s", Http, Uri); else - my_snprintf(buf, sizeof(buf)-1, "%s/%s", Http, Uri); + my_snprintf(buf, bufsz, "%s/%s", Http, Uri); } else - my_snprintf(buf, sizeof(buf)-1, "%s", Http); + my_snprintf(buf, bufsz, "%s", Http); - FILE *f= fopen(Fn, "wb"); if ((curl_res= curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errbuf)) != CURLE_OK || (curl_res= curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, @@ -271,15 +297,18 @@ int RESTDEF::curl_run(PGLOBAL g) "curl returned this error code: %u " "with the following error message: %s", curl_res, curl_errbuf[0] ? curl_errbuf : curl_easy_strerror(curl_res)); + fclose(f); + free(buf); return 1; } } curl_easy_cleanup(curl); + free(buf); fclose(f); bool is_error = http_code < 200 || http_code >= 300; if (is_error) { - snprintf(g->Message, sizeof(g->Message), "server error"); + snprintf(g->Message, sizeof(g->Message), "Server returned error code %ld", http_code); return 1; } return 0; From 0dadb978887d0a098b6870075580a5c7f6bd237f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 14 May 2026 17:00:37 +1000 Subject: [PATCH 6/7] connect MemoryStruct unneeded --- storage/connect/tabrest.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/storage/connect/tabrest.h b/storage/connect/tabrest.h index 970916ef76763..08ad1a37cccdc 100644 --- a/storage/connect/tabrest.h +++ b/storage/connect/tabrest.h @@ -18,15 +18,6 @@ PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char* tab, char* db, bool info); #endif // !MARIADB -/***********************************************************************/ -/* Data structure for curl callback function */ -/***********************************************************************/ -struct MemoryStruct { - char *memory; - size_t size; -}; - - /***********************************************************************/ /* Restest table. */ /***********************************************************************/ From 79c9f470d7c5483cd46d320b4676de036ccf9535 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 14 May 2026 17:18:18 +1000 Subject: [PATCH 7/7] connect: link order - zlib after rest(curl) Though zlib was for CONNECT_WITH_VCT mainly. --- storage/connect/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 7398d26538106..a89bd76416060 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -396,8 +396,8 @@ MYSQL_ADD_PLUGIN(connect ${CONNECT_SOURCES} STORAGE_ENGINE COMPONENT connect-engine RECOMPILE_FOR_EMBEDDED - LINK_LIBRARIES ${ZLIB_LIBRARIES} ${XML_LIBRARY} ${ICONV_LIBRARY} - ${ODBC_LIBRARY} ${MONGOC_LIBRARY} ${IPHLPAPI_LIBRARY} ${REST_LIBRARY}) + LINK_LIBRARIES ${XML_LIBRARY} ${ICONV_LIBRARY} + ${ODBC_LIBRARY} ${MONGOC_LIBRARY} ${IPHLPAPI_LIBRARY} ${REST_LIBRARY} ${ZLIB_LIBRARIES}) IF(NOT TARGET connect) RETURN()