OpenEuphoria

1. http_get.e

• Posted by Senator Nov 21, 2018
• 1418 views
• Last edited Nov 26, 2018

With Pete's permission I have endeavored to excerpt his http_get code from WEE 0.52's updater.ex into a stand alone include file, http_get.e.. Pete gave me some pointers and the code seems to work well under LINUX. I left Pete's Windows routine untouched. I hope I haven't mucked up Pete's code too much. Suggestions and ideas for improvement are welcome.

Thanks again, Pete!
Regards,
Ken Rhodes

-- 
--  https_get.e  
-- 
--  code extracted and modifed from Pete Eberleins WEE .52 updater.ex: 
--  Copyright (c) 1998-2018  Pete Eberlein 
--  
-- Permission is hereby granted, free of charge, to any person obtaining a copy 
-- of this software and associated documentation files (the "Software"), to deal 
-- in the Software without restriction, including without limitation the rights 
-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 
-- copies of the Software, and to permit persons to whom the Software is 
-- furnished to do so, subject to the following conditions: 
--  
-- The above copyright notice and this permission notice shall be included in 
-- all copies or substantial portions of the Software. 
--  
-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
-- THE SOFTWARE. 
 
--  minor LINUX modifications by Kenneth Rhodes: 
-------------------------------------------------------------------------------------------------------- 
--  supplementary routines: 
--  extract_fn(sequence url) 
--  write_file_data(sequence url) 
 
include std/console.e 
--include std/net/http.e  -- doesn't support https 
include std/io.e 
include std/get.e 
include std/filesys.e 
include std/hash.e 
include std/dll.e 
include std/machine.e 
include std/sequence.e 
 
 
ifdef WINDOWS then 
-- 
-- Implement http_get via WinINet for Windows 
-- 
 
constant 
    wininet = open_dll("wininet.dll") 
if wininet = 0 then 
    puts(1, "Failed to open wininet.dll\n") 
    abort(1) 
end if 
 
constant 
    InternetOpen = define_c_func(wininet, "InternetOpenA", {C_POINTER, C_DWORD, C_POINTER, C_POINTER, C_DWORD}, C_HANDLE), 
    InternetCloseHandle = define_c_func(wininet, "InternetCloseHandle", {C_HANDLE}, C_BOOL), 
    InternetOpenUrl = define_c_func(wininet, "InternetOpenUrlA", {C_HANDLE, C_POINTER, C_POINTER, C_DWORD, C_DWORD, C_POINTER}, C_HANDLE), 
    InternetReadFile = define_c_func(wininet, "InternetReadFile", {C_HANDLE, C_POINTER, C_DWORD, C_POINTER}, C_BOOL) 
if InternetOpen = -1 or 
   InternetCloseHandle = -1 or 
   InternetOpenUrl = -1 or 
   InternetReadFile = -1 then 
    puts(1, "Failed to find functions in wininet\n") 
    abort(1) 
end if 
 
constant 
    INTERNET_OPEN_TYPE_PRECONFIG = 0 
 
public function https_get(sequence url) 
    atom agent_ptr = allocate_string("Mozilla/4.0 (compatible)") 
    atom url_ptr = allocate_string(url) 
    atom ih, ch -- internet handle, connection handle 
    integer bufsize = 4096 
    atom buf_ptr = allocate(bufsize) 
    atom bytesread_ptr = allocate(4) -- LPDWORD 
    object res = -1 
 
    ih = c_func(InternetOpen, {agent_ptr, INTERNET_OPEN_TYPE_PRECONFIG, NULL, NULL, 0}) 
    if ih then 
        ch = c_func(InternetOpenUrl, {ih, url_ptr, NULL, 0, 0, NULL}) 
        if ch then 
            res = "" 
            while c_func(InternetReadFile, {ch, buf_ptr, bufsize, bytesread_ptr}) != 1 or 
                  peek4u(bytesread_ptr) != 0 do 
                res &= peek({buf_ptr, peek4u(bytesread_ptr)}) 
            end while 
--            res = {{{"Status","200"}}, res} 
            c_func(InternetCloseHandle, {ih}) 
        end if 
        c_func(InternetCloseHandle, {ih}) 
    end if 
 
    free(agent_ptr) 
    free(url_ptr) 
    free(buf_ptr) 
    free(bytesread_ptr) 
 
    return res 
end function 
 
elsedef 
 
-- 
-- Implement https_get via libcurl for UNIX 
-- 
 
integer libcurl = open_dll("libcurl.so") 
if libcurl = 0 then libcurl = open_dll("libcurl.so.3") end if 
if libcurl = 0 then libcurl = open_dll("libcurl.so.4") end if 
if libcurl = 0 then  puts(1, "Failed to open libcurl.so\n")  abort(1) end if 
 
constant 
    CURLOPT_URL = 10002, 
    CURLOPT_WRITEFUNCTION = 20011, 
    CURLOPT_WRITEDATA = 10001, 
    CURLOPT_FOLLOWLOCATION = 52 
 
sequence cb_data 
function curl_callback(atom ptr, atom size, atom nmemb, atom writedata) 
    cb_data &= peek({ptr, size * nmemb}) 
    return nmemb 
end function 
constant curl_cb = call_back(routine_id("curl_callback")) 
 
public function https_get(sequence url) 
atom url_ptr = allocate_string(url), res, curl 
integer curl_easy_init = define_c_func(libcurl, "curl_easy_init", {}, C_POINTER), 
    curl_easy_setopt =   define_c_proc(libcurl, "curl_easy_setopt", {C_POINTER, C_LONG, C_POINTER}), 
    curl_easy_perform =define_c_func(libcurl, "curl_easy_perform", {C_POINTER}, C_LONG), 
    curl_easy_cleanup =define_c_proc(libcurl, "curl_easy_cleanup", {C_POINTER}) 
    curl = c_func(curl_easy_init, {}) 
    if not curl then  
        fail("\nFailed to init libcurl\n")  
    return -1  
    end if 
    if not url_ptr or 
        curl_easy_init = -1 or    
        curl_easy_setopt = -1 or 
        curl_easy_perform = -1 or    
        curl_easy_cleanup = -1 then 
        fail("\nFailed to find functions in libcurl\n") 
    return -1  
    end if 
    c_proc(curl_easy_setopt, {curl, CURLOPT_WRITEFUNCTION, curl_cb}) 
    c_proc(curl_easy_setopt, {curl, CURLOPT_WRITEDATA, 0}) 
    c_proc(curl_easy_setopt, {curl, CURLOPT_FOLLOWLOCATION, 1}) 
    cb_data = "" 
    c_proc(curl_easy_setopt, {curl, CURLOPT_URL, url_ptr}) 
    res = c_func(curl_easy_perform, {curl}) 
    free(url_ptr)     
    c_proc(curl_easy_cleanup, {curl})  
    if res = 0 then  
         if match("Error", sprintf("%s",{cb_data})) or 
         match("404 Not Found", sprintf("%s",{cb_data}))then 
            fail(cb_data) 
        else 
            return cb_data  
        end if 
    end if 
return -1 
end function 
 
end ifdef  
 
public function extract_fn(sequence url) 
-- extract a file name from a url 
integer z = length(url) 
sequence fn="" 
    for i = z  to 1 by -1  do 
        if equal({SLASH}, sprintf("%s",url[i])) then  
            fn = slice(url, i+1, z) 
            exit  
        end if 
    end for 
return fn 
end function 
 
function fail(sequence msg) 
sequence txt = "\n\nPress any key to exit\n" 
    display(msg & txt) 
    wait_key() 
return 0 
end function 
 
public function write_file_data(sequence url)  
-- returns 1 if write_file_data succeeds  
-- returns 0 otherwise 
integer exit_code = 1 
object data 
sequence fn = extract_fn(url) 
     if length(fn) then  
        data = https_get(url)  
        if atom(data) then 
            exit_code = fail(sprintf("\nFailed to download file: %s\n", {fn})) 
        else 
            if write_file(fn,data) = -1 then 
               exit_code = fail(sprintf("\nFailed to write_file %s\n", {fn}))  
            else  
                display(sprintf("\nwrite_file_data %s successfull\n", {fn}))  
            end if 
        end if 
    else  
       exit_code = fail(sprintf("\nFailed to extract a file name from: %s", {url})) 
    end if 
return exit_code 
end function 
 
-- test 
-- constant archive = "https://archive.usingeuphoria.com/" 
-- write_file_data(archive & "euhelp.tar")  
-- displays error message: file not downloaded 
-- write_file_data(archive & "euhelper.tar") 
-- write_file_data(archive & "simple.tar") 
 
 
 

3. Re: http_get.e -- LINUX CODE

• Posted by SDPringle Nov 22, 2018
• 1365 views

Some might wonder why not use std/net/http.e? The std/net/http.e is rather useless and it is a good idea to use a CURL library for this. The standard library routines uses HTTP only and the world has upgraded to HTTPS. You could adapt the included Euphoria unit tests for this code. Also, it wouldn't be such a bad idea to adopt this for std/net/http.

S D Pringle

5. Re: https_get.e

• Posted by Senator Nov 23, 2018
• 1354 views
• Last edited Nov 26, 2018

I got carried away thinking maybe the file and functions should be called "https_get". Also, I changed the procedures to functions thinking that perhaps an include file should not call abort().

https_get(url) now catches and displays "404 Not Found" found error without writing a "dead" file. Commented out the Windows line specific to "updater.ex" usage.

-- 
--  https_get.e  
-- 
--  code extracted and modifed from Pete Eberleins WEE .52 updater.ex: 
--  Copyright (c) 1998-2018  Pete Eberlein 
--  
-- Permission is hereby granted, free of charge, to any person obtaining a copy 
-- of this software and associated documentation files (the "Software"), to deal 
-- in the Software without restriction, including without limitation the rights 
-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 
-- copies of the Software, and to permit persons to whom the Software is 
-- furnished to do so, subject to the following conditions: 
--  
-- The above copyright notice and this permission notice shall be included in 
-- all copies or substantial portions of the Software. 
--  
-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
-- THE SOFTWARE. 
 
--  minor LINUX modifications by Kenneth Rhodes: 
-------------------------------------------------------------------------------------------------------- 
--  supplementary routines: 
--  extract_fn(sequence url) 
--  write_file_data(sequence url) 
 
include std/console.e 
--include std/net/http.e  -- doesn't support https 
include std/io.e 
include std/get.e 
include std/filesys.e 
include std/hash.e 
include std/dll.e 
include std/machine.e 
include std/sequence.e 
 
 
ifdef WINDOWS then 
-- 
-- Implement http_get via WinINet for Windows 
-- 
 
constant 
    wininet = open_dll("wininet.dll") 
if wininet = 0 then 
    puts(1, "Failed to open wininet.dll\n") 
    abort(1) 
end if 
 
constant 
    InternetOpen = define_c_func(wininet, "InternetOpenA", {C_POINTER, C_DWORD, C_POINTER, C_POINTER, C_DWORD}, C_HANDLE), 
    InternetCloseHandle = define_c_func(wininet, "InternetCloseHandle", {C_HANDLE}, C_BOOL), 
    InternetOpenUrl = define_c_func(wininet, "InternetOpenUrlA", {C_HANDLE, C_POINTER, C_POINTER, C_DWORD, C_DWORD, C_POINTER}, C_HANDLE), 
    InternetReadFile = define_c_func(wininet, "InternetReadFile", {C_HANDLE, C_POINTER, C_DWORD, C_POINTER}, C_BOOL) 
if InternetOpen = -1 or 
   InternetCloseHandle = -1 or 
   InternetOpenUrl = -1 or 
   InternetReadFile = -1 then 
    puts(1, "Failed to find functions in wininet\n") 
    abort(1) 
end if 
 
constant 
    INTERNET_OPEN_TYPE_PRECONFIG = 0 
 
public function https_get(sequence url) 
    atom agent_ptr = allocate_string("Mozilla/4.0 (compatible)") 
    atom url_ptr = allocate_string(url) 
    atom ih, ch -- internet handle, connection handle 
    integer bufsize = 4096 
    atom buf_ptr = allocate(bufsize) 
    atom bytesread_ptr = allocate(4) -- LPDWORD 
    object res = -1 
 
    ih = c_func(InternetOpen, {agent_ptr, INTERNET_OPEN_TYPE_PRECONFIG, NULL, NULL, 0}) 
    if ih then 
        ch = c_func(InternetOpenUrl, {ih, url_ptr, NULL, 0, 0, NULL}) 
        if ch then 
            res = "" 
            while c_func(InternetReadFile, {ch, buf_ptr, bufsize, bytesread_ptr}) != 1 or 
                  peek4u(bytesread_ptr) != 0 do 
                res &= peek({buf_ptr, peek4u(bytesread_ptr)}) 
            end while 
--            res = {{{"Status","200"}}, res} 
            c_func(InternetCloseHandle, {ih}) 
        end if 
        c_func(InternetCloseHandle, {ih}) 
    end if 
 
    free(agent_ptr) 
    free(url_ptr) 
    free(buf_ptr) 
    free(bytesread_ptr) 
 
    return res 
end function 
 
elsedef 
 
-- 
-- Implement https_get via libcurl for UNIX 
-- 
 
integer libcurl = open_dll("libcurl.so") 
if libcurl = 0 then libcurl = open_dll("libcurl.so.3") end if 
if libcurl = 0 then libcurl = open_dll("libcurl.so.4") end if 
if libcurl = 0 then  puts(1, "Failed to open libcurl.so\n")  abort(1) end if 
 
constant 
    CURLOPT_URL = 10002, 
    CURLOPT_WRITEFUNCTION = 20011, 
    CURLOPT_WRITEDATA = 10001, 
    CURLOPT_FOLLOWLOCATION = 52 
 
sequence cb_data 
function curl_callback(atom ptr, atom size, atom nmemb, atom writedata) 
    cb_data &= peek({ptr, size * nmemb}) 
    return nmemb 
end function 
constant curl_cb = call_back(routine_id("curl_callback")) 
 
public function https_get(sequence url) 
atom url_ptr = allocate_string(url), res, curl 
integer curl_easy_init = define_c_func(libcurl, "curl_easy_init", {}, C_POINTER), 
    curl_easy_setopt =   define_c_proc(libcurl, "curl_easy_setopt", {C_POINTER, C_LONG, C_POINTER}), 
    curl_easy_perform =define_c_func(libcurl, "curl_easy_perform", {C_POINTER}, C_LONG), 
    curl_easy_cleanup =define_c_proc(libcurl, "curl_easy_cleanup", {C_POINTER}) 
    curl = c_func(curl_easy_init, {}) 
    if not curl then  
        fail("\nFailed to init libcurl\n")  
    return -1  
    end if 
    if not url_ptr or 
        curl_easy_init = -1 or    
        curl_easy_setopt = -1 or 
        curl_easy_perform = -1 or    
        curl_easy_cleanup = -1 then 
        fail("\nFailed to find functions in libcurl\n") 
    return -1  
    end if 
    c_proc(curl_easy_setopt, {curl, CURLOPT_WRITEFUNCTION, curl_cb}) 
    c_proc(curl_easy_setopt, {curl, CURLOPT_WRITEDATA, 0}) 
    c_proc(curl_easy_setopt, {curl, CURLOPT_FOLLOWLOCATION, 1}) 
    cb_data = "" 
    c_proc(curl_easy_setopt, {curl, CURLOPT_URL, url_ptr}) 
    res = c_func(curl_easy_perform, {curl}) 
    free(url_ptr)     
    c_proc(curl_easy_cleanup, {curl})  
    if res = 0 then  
         if match("Error", sprintf("%s",{cb_data})) or 
         match("404 Not Found", sprintf("%s",{cb_data}))then 
            fail(cb_data) 
        else 
            return cb_data  
        end if 
    end if 
return -1 
end function 
 
end ifdef  
 
public function extract_fn(sequence url) 
-- extract a file name from a url 
integer z = length(url) 
sequence fn="" 
    for i = z  to 1 by -1  do 
        if equal({SLASH}, sprintf("%s",url[i])) then  
            fn = slice(url, i+1, z) 
            exit  
        end if 
    end for 
return fn 
end function 
 
function fail(sequence msg) 
sequence txt = "\n\nPress any key to exit\n" 
    display(msg & txt) 
    wait_key() 
return 0 
end function 
 
public function write_file_data(sequence url)  
-- returns 1 if write_file_data succeeds  
-- returns 0 otherwise 
integer exit_code = 1 
object data 
sequence fn = extract_fn(url) 
     if length(fn) then  
        data = https_get(url)  
        if atom(data) then 
            exit_code = fail(sprintf("\nFailed to download file: %s\n", {fn})) 
        else 
            if write_file(fn,data) = -1 then 
               exit_code = fail(sprintf("\nFailed to write_file %s\n", {fn}))  
            else  
                display(sprintf("\nwrite_file_data %s successfull\n", {fn}))  
            end if 
        end if 
    else  
       exit_code = fail(sprintf("\nFailed to extract a file name from: %s", {url})) 
    end if 
return exit_code 
end function 
 
-- test 
-- constant archive = "https://archive.usingeuphoria.com/" 
-- write_file_data(archive & "euhelp.tar")  
-- displays error message: file not downloaded 
-- write_file_data(archive & "euhelper.tar") 
-- write_file_data(archive & "simple.tar") 
 

7. Re: https_get.e

• Posted by SDPringle Nov 26, 2018
• 1227 views

I bet you guys could create a https_post too, eh?