OpenEuphoria

1. HTTPS with OpenSSL

• Posted by jmduro Jul 15, 2016
• 3273 views

Here is an attempt to let OpenEuphoria manage HTTPS with help of OpenSSL. OpenSSL Windows binaries (openssl.exe, libeay32.dll and ssleay32.dll) have to be in the same directory as following source code.

I get something, but I'm not fully sure it works. Help needed.

• std/https.e

--**** 
-- == HTTPS Client 
-- 
-- <<LEVELTOC level=2 depth=4>> 
-- 
 
namespace https 
 
include std/base64.e 
include std/convert.e 
include std/rand.e 
include std/sequence.e 
include std/text.e 
include std/types.e 
include std/pipeio.e as pipe 
 
 
include std/net/dns.e 
include std/net/url.e as url 
 
include euphoria/info.e 
 
constant USER_AGENT_HEADER =  
	sprintf("User-Agent: Euphoria-HTTP/%d.%d\r\n", { 
		version_major(), version_minor() }) 
 
enum R_HOST, R_PORT, R_PATH, R_REQUEST 
constant FR_SIZE = 4 
 
--**** 
-- === Error Codes 
-- 
 
public enum by -1 
	ERR_MALFORMED_URL = -1, 
	ERR_INVALID_PROTOCOL, 
	ERR_INVALID_DATA, 
	ERR_INVALID_DATA_ENCODING, 
	ERR_HOST_LOOKUP_FAILED, 
	ERR_CONNECT_FAILED, 
	ERR_SEND_FAILED, 
	ERR_RECEIVE_FAILED 
 
--**** 
-- === Constants 
 
public enum 
	FORM_URLENCODED, 
	MULTIPART_FORM_DATA 
	 
public enum 
	--** 
	-- No encoding is necessary 
	ENCODE_NONE = 0, 
	--** 
	-- Use Base64 encoding 
	ENCODE_BASE64 
 
constant ENCODING_STRINGS = { 
	"application/x-www-form-urlencoded", 
	"multipart/form-data" 
} 
 
object command_pipe=-1, openssl_process=-1 
-- 
-- returns: { host, port, path, base_reqest } 
-- 
 
function format_base_request(sequence request_type, sequence url, object headers) 
	sequence request = "" 
	sequence formatted_request 
	integer noport = 0 
 
	object parsedUrl = url:parse(url) 
	if atom(parsedUrl) then 
		return ERR_MALFORMED_URL 
	elsif not equal(parsedUrl[URL_PROTOCOL], "https") then 
		return ERR_INVALID_PROTOCOL 
	end if 
 
	sequence host = parsedUrl[URL_HOSTNAME] 
 
	integer port = parsedUrl[URL_PORT] 
	if port = 0 then 
		port = 80 
		noport = 1 
	end if 
 
	sequence path 
	if sequence(parsedUrl[URL_PATH]) then 
		path = parsedUrl[URL_PATH] 
	else 
		path = "/" 
	end if 
 
	if sequence(parsedUrl[URL_QUERY_STRING]) then 
		path &= "?" & parsedUrl[URL_QUERY_STRING] 
	end if 
 
	-- only specify the port in the request if the caller did so explicitly 
	-- some sites, such as euphoria.pastey.net, will break otherwise 
	if noport then 
		request = sprintf("%s %s HTTP/1.0\r\nHost: %s\r\n", { 
			request_type, url, path, host }) 
	else 
		request = sprintf("%s %s HTTP/1.0\r\nHost: %s:%d\r\n", { 
			request_type, url, host, port }) 
	end if 
 
	integer has_user_agent = 0 
	integer has_connection = 0 
 
	if sequence(headers) then 
		for i = 1 to length(headers) do 
			object header = headers[i] 
			if equal(header[1], "User-Agent") then 
				has_user_agent = 1 
			elsif equal(header[1], "Connection") then 
				has_connection = 1 
			end if 
 
			request &= sprintf("%s: %s\r\n", header) 
		end for 
	end if 
 
	if not has_user_agent then 
		request &= USER_AGENT_HEADER 
	end if 
	if not has_connection then 
		request &= "Connection: close\r\n" 
	end if 
 
	formatted_request = repeat(0, FR_SIZE) 
	formatted_request[R_HOST] = host 
	formatted_request[R_PORT] = port 
	formatted_request[R_PATH] = path 
	formatted_request[R_REQUEST] = request 
	return formatted_request 
end function 
 
-- 
-- encode a sequence of key/value pairs 
-- 
 
function form_urlencode(sequence kvpairs) 
	sequence data = "" 
 
	for i = 1 to length(kvpairs) do 
		object kvpair = kvpairs[i] 
 
		if i > 1 then 
			data &= "&" 
		end if 
 
		data &= kvpair[1] & "=" & url:encode(kvpair[2]) 
	end for 
 
	return data 
end function 
 
function multipart_form_data_encode(sequence kvpairs, sequence boundary) 
	sequence data = "" 
	 
	for i = 1 to length(kvpairs) do 
		object kvpair = kvpairs[i] 
 
		integer enctyp = ENCODE_NONE 
		sequence mimetyp = "" 
		 
		if i > 1 then 
			data &= "\r\n" 
		end if 
		 
		data &= "--" & boundary & "\r\n" 
		data &= "Content-Disposition: form-data; name=\"" & kvpair[1] & "\"" 
		if length(kvpair) = 5 then 
			data &= "; filename=\"" & kvpair[3] & "\"\r\n" 
			data &= "Content-Type: " & kvpair[4] & "\r\n" 
			 
			switch kvpair[5] do 
				case ENCODE_NONE then 
				case ENCODE_BASE64 then 
					data &= "Content-Transfer-Encoding: base64\r\n" 
					kvpair[2] = base64:encode(kvpair[2], 76) 
			end switch 
		else 
			data &= "\r\n" 
		end if	 
			 
		data &= "\r\n" & kvpair[2] 
	end for 
 
	return data & "\r\n--" & boundary & "--" 
end function 
 
constant rand_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" 
constant rand_chars_len = length(rand_chars) 
 
function random_boundary(integer len) 
	sequence boundary = repeat(0, len) 
	 
	for i = 1 to len do 
		boundary[i] = rand_chars[rand(rand_chars_len)] 
	end for 
	 
	return boundary 
end function 
 
public function run_openssl(sequence dest) 
  command_pipe = pipe:create() 
  openssl_process = pipe:exec("openssl s_client -connect " & dest, command_pipe) 
  if atom(openssl_process) then 
    printf(1, "Failed to exec() with error %x", pipe:error_no()) 
    pipe:kill(openssl_process) 
    return 0 
  end if 
  return 1 
end function 
 
function write_openssl(sequence data) 
  return pipe:write(openssl_process[STDIN], data) 
end function 
 
function read_openssl() 
  sequence data = "" 
  object c = pipe:read(openssl_process[pipe:STDOUT], 256) 
  while sequence(c) and length(c) do 
    data &= c 
    if atom(c) then 
      printf(1, "Failed on read with error %x", pipe:error_no()) 
      pipe:kill(openssl_process) 
      return 0 
    end if 
    c = pipe:read(openssl_process[pipe:STDOUT], 256) 
  end while 
  return data 
end function 
 
public procedure close_openssl() 
  pipe:kill(openssl_process) 
end procedure 
 
-- 
-- Send an HTTPS request 
-- 
 
function execute_request(sequence request, integer timeout) 
	if write_openssl(request) != length(request) then 
		return ERR_SEND_FAILED 
	end if 
 
	atom start_time = time() 
	integer got_header = 0, content_length = 0 
	sequence content = "" 
	sequence headers = {} 
	while time() - start_time < timeout label "top" do 
		if got_header and length(content) = content_length then 
			exit 
		end if 
 
		object data = read_openssl() 
		if atom(data) then 
			return ERR_RECEIVE_FAILED 
		end if 
		if length(data) = 0 then 
			-- zero bytes received, we the 'data' waiting was 
			-- a disconnect. 
			exit "top" 
  		end if 
 
		content &= data 
 
		if not got_header then 
			integer header_end_pos = match("\r\n\r\n", content) 
			if header_end_pos then 
				-- we have a header, let's parse it and figure out 
				-- the content length. 
				sequence raw_header = content[1..header_end_pos] 
				content = content[header_end_pos + 4..$] 
 
				sequence header_lines = split(raw_header, "\r\n") 
				headers = append(headers, split(header_lines[1], " ")) 
				for i = 2 to length(header_lines) do 
					object header = header_lines[i] 
					sequence this_header = split(header, ": ", , 1) 
					this_header[1] = lower(this_header[1]) 
					headers = append(headers, this_header) 
 
					if equal(this_header[1], "content-length") then 
						content_length = to_number(this_header[2]) 
					end if 
				end for 
 
				got_header = 1 
			end if 
		end if 
	end while 
 
	return { headers, content } 
end function 
 
--**** 
-- === Get/Post Routines 
 
--** 
-- Post data to a HTTP resource. 
-- 
-- Parameters: 
--   * ##url##     - URL to send post request to 
--   * ##data##    - Form data (described later) 
--   * ##headers## - Additional headers added to request 
--   * ##follow_redirects## - Maximum redirects to follow 
--   * ##timeout## - Maximum number of seconds to wait for a response 
-- 
-- Returns: 
--   An integer error code or a 2 element sequence. Element 1 is a sequence 
--   of key/value pairs representing the result header information. element 
--   2 is the body of the result. 
-- 
--   If result is a negative integer, that represents a local error condition. 
-- 
--   If result is a positive integer, that represents a HTTP error value from 
--   the server. 
-- 
-- Data Sequence: 
--  This sequence should contain key value pairs representing the expected form 
--  elements of the called URL. For a simple url-encoded form: 
-- 
--  { {"name", "John Doe"}, {"age", "22"}, {"city", "Small Town"}} 
 
-- 
--  All Keys and Values should be a sequence. 
-- 
--  If the post requires multipart form encoding then the sequence is a little 
--  different. The first element of the data sequence must be [[:MULTIPART_FORM_DATA]]. 
--  All subsequent field values should be key/value pairs as described above **except** 
--  for a field representing a file upload. In that case the sequence should be: 
-- 
--  ##{ FIELD-NAME, FILE-VALUE, FILE-NAME, MIME-TYPE, ENCODING-TYPE }## 
-- 
--  Encoding type can be 
--    * [[:ENCODE_NONE]] 
--    * [[:ENCODE_BASE64]] 
-- 
--  An example for a multipart form encoded post request data sequence 
-- 
--  {  
--    { "name", "John Doe" },  
--    { "avatar", file_content, "me.png", "image/png", ENCODE_BASE64 }, 
--    { "city", "Small Town" } 
--  } 
 
-- 
-- See Also: 
--   [[:http_get]] 
-- 
 
public function https_post(sequence url, object data, object headers = 0, 
		integer follow_redirects = 10, integer timeout = 15) 
		 
	follow_redirects = follow_redirects -- Not used yet. 
	 
	if not sequence(data) or length(data) = 0 then 
		return ERR_INVALID_DATA 
	end if 
 
	object request = format_base_request("POST", url, headers) 
	if atom(request) then 
		return request 
	end if 
 
	integer data_type 
	if ascii_string(data) or sequence(data[1]) then 
		data_type = FORM_URLENCODED 
	else 
		if data[1] < 1 or data[1] > 2 then 
			return ERR_INVALID_DATA_ENCODING 
		end if 
 
		data_type = data[1] 
		data = data[2] 
	end if 
 
	-- data now contains either a string sequence already encoded or 
	-- a sequence of key/value pairs to be encoded. We know the length 
	-- is greater than 0, so check the first element to see if it's a 
	-- sequence or an atom. That will tell us what we have. 
	-- 
	-- If we have key/value pairs then we will need to encode that data 
	-- according to our data_type. 
 
	sequence content_type = ENCODING_STRINGS[data_type] 
	if sequence(data[1]) then 
		-- We have key/value pairs 
		if data_type = FORM_URLENCODED then 
			data = form_urlencode(data) 
		else 
			sequence boundary = random_boundary(20) 
			content_type &= "; boundary=" & boundary 
			data = multipart_form_data_encode(data, boundary) 
		end if 
	end if 
 
	request[R_REQUEST] &= sprintf("Content-Type: %s\r\n", { content_type }) 
	request[R_REQUEST] &= sprintf("Content-Length: %d\r\n", { length(data) }) 
	request[R_REQUEST] &= "\r\n" 
	request[R_REQUEST] &= data 
 
	return execute_request(request[R_REQUEST], timeout) 
end function 
 
--** 
-- Get a HTTP resource. 
-- 
-- Returns: 
--   An integer error code or a 2 element sequence. Element 1 is a sequence 
--   of key/value pairs representing the result header information. Element 
--   2 is the body of the result. 
-- 
--   If result is a negative integer, that represents a local error condition. 
-- 
--   If result is a positive integer, that represents a HTTP error value from 
--   the server. 
-- 
-- Example: 
-- 
-- include std/console.e -- for display() 
-- include std/net/http.e 
-- 
-- object result = http_get("http://example.com")  
-- if atom(result) then  
--    printf(1, "Web error: %d\n", result)  
--     abort(1)  
-- end if  
--  
-- display(result[1]) -- header key/value pairs 
-- printf(1, "Content: %s\n", { result[2] })  
 
-- 
-- See Also: 
--   [[:http_post]] 
-- 
 
public function https_get(sequence url, object headers = 0, integer follow_redirects = 10, 
		integer timeout = 15) 
	object request = format_base_request("GET", url, headers) 
	 
	follow_redirects = follow_redirects -- Not used yet. 
	 
	if atom(request) then 
		return request 
	end if 
 
	-- No more work necessary, terminate the request with our ending CR LF 
	request[R_REQUEST] &= "\r\n" 
 
	return execute_request(request[R_REQUEST], timeout) 
end function 

• test program

include std/types.e 
include std/console.e 
include std/io.e 
include std/https.e 
 
integer offset = 0 
 
procedure dumpHeader(object x, sequence Name, integer Output=1, sequence path={}, integer level=0, sequence prefix="") 
-- prints a sequence structure in a human readable way 
  integer subSequence 
  sequence s, offset 
 
  if level=0 then printf(Output, "%s = ", {Name}) end if 
  offset = "" 
  for i = 1 to level do offset &= ".  " end for 
  s = "" 
  if t_display(x) then 
    if length(x) = 0 then 
      s = " \"\"" 
    else 
      s = sprintf(" \"%s\"", {x}) 
    end if 
    puts(Output, offset&prefix&s&"\n") 
  else 
    subSequence = 0 
    for i=1 to length(x) do 
      if sequence(x[i]) then 
        subSequence = 1 
        exit 
      end if 
    end for 
    if subSequence = 0 then 
      s = sprintf(" %s", {x}) 
      puts(Output, offset&prefix&s&"\n") 
    else 
      if length(s) = 0 then 
        puts(Output, offset&prefix&"\n") 
      end if 
      for i=1 to length(x) do 
        prefix = sprintf("[%d]", i) 
        dumpHeader(x[i], Name, Output, path&sprintf("[%d]", i), level+1, prefix) 
      end for 
    end if 
  end if 
  if Output > 2 then flush(Output) end if 
end procedure 
 
  if run_openssl("gw.geneanet.org:443") then 
--  sequence s = read_openssl() 
    object result = https_get("https://gw.geneanet.org/connexion/")  
    if atom(result) then  
      printf(1, "Web error: %d\n", result)  
      maybe_any_key() 
      abort(1)  
    end if  
    dumpHeader(result[1], "header") 
    puts(1, result[2] & "\n") 
    close_openssl() 
  end if 
 
maybe_any_key() 

Jean-Marc

3. Re: HTTPS with OpenSSL

• Posted by ghaberek (admin) Jul 18, 2016
• 3088 views

Ugh. I forgot how craptacular the CURLOPT macros are. The authors even refer to this as macro-mania.

/* 
 * This macro-mania below setups the CURLOPT_[what] enum, to be used with 
 * curl_easy_setopt(). The first argument in the CINIT() macro is the [what] 
 * word. 
 */ 
 
typedef enum { 
  /* This is the FILE * or void * the regular output should be written to. */ 
  CINIT(WRITEDATA, OBJECTPOINT, 1), 
 
  /* The full URL to get/put */ 
  CINIT(URL, STRINGPOINT, 2), 
 
  /* Port number to connect to, if other than default. */ 
  CINIT(PORT, LONG, 3), 
 
  /* Name of proxy to use. */ 
  CINIT(PROXY, STRINGPOINT, 4), 
 
  ... 

-Greg

5. Re: HTTPS with OpenSSL

• Posted by ghaberek (admin) Jul 18, 2016
• 3064 views

This is what I came up with. I translated all of the macro-mania with the Regex find/replace in Notepad++.

map m_CURLopttype = map:new_from_kvpairs({ 
    { "CURLOPTTYPE_LONG",          CURLOPTTYPE_LONG }, 
    { "CURLOPTTYPE_OBJECTPOINT",   CURLOPTTYPE_OBJECTPOINT }, 
    { "CURLOPTTYPE_STRINGPOINT",   CURLOPTTYPE_OBJECTPOINT }, 
    { "CURLOPTTYPE_FUNCTIONPOINT", CURLOPTTYPE_FUNCTIONPOINT }, 
    { "CURLOPTTYPE_OFF_T",         CURLOPTTYPE_OFF_T } 
}) 
 
constant LONG          = "CURLOPTTYPE_LONG" 
constant OBJECTPOINT   = "CURLOPTTYPE_OBJECTPOINT" 
constant STRINGPOINT   = "CURLOPTTYPE_OBJECTPOINT" 
constant FUNCTIONPOINT = "CURLOPTTYPE_FUNCTIONPOINT" 
constant OFF_T         = "CURLOPTTYPE_OFF_T" 
 
map m_CURLoption  = map:new() 
 
function CINIT( sequence na, sequence t, atom nu ) 
 
    atom value = map:get( m_CURLopttype, t, 0 ) + nu 
    map:put( m_CURLoption, "CURLOPT_" & na, {na,t,nu} ) 
 
    return value 
end function 
 
public constant 
 
    /* This is the FILE * or void * the regular output should be written to. */ 
    CURLOPT_WRITEDATA = CINIT("WRITEDATA", OBJECTPOINT, 1), 
 
    /* The full URL to get/put */ 
    CURLOPT_URL = CINIT("URL", STRINGPOINT, 2), 
 
    /* Port number to connect to, if other than default. */ 
    CURLOPT_PORT = CINIT("PORT", LONG, 3), 
 
    /* Name of proxy to use. */ 
    CURLOPT_PROXY = CINIT("PROXY", STRINGPOINT, 4), 
 
... 

Not sure how necessary this approach is when all I need are constant values...

-Greg

7. Re: HTTPS with OpenSSL

• Posted by jmduro Aug 05, 2016
• 2934 views

Maybe using an external DLL is a better way than calling an external process, but I built a libcurl wrapper from the 7.50.1 release and it runs short: it exits without any error on curl_global_init(CURL_GLOBAL_DEFAULT) which has to be the first call.

Jean-Marc

9. Re: HTTPS with OpenSSL

• Posted by jmduro Aug 06, 2016
• 2951 views

Thanks a lot Greg!

No I didn't prefix function names with a plus. Now the first example works. I cant get the page content of https://example.com.

I have to transcript many more C examples before I can provide the source code. Some functions have not been ported. Only a subset of the easy branch has been ported.

Jean-Marc

11. Re: HTTPS with OpenSSL

• Posted by jmduro Aug 10, 2016
• 2786 views

I tried another approach by adding missing functions to Fabio Ramirez's wrapper for Wininet.

Source code is here: http://jean-marc.duro.pagesperso-orange.fr/EInetLib_full.ew

I ran following code:

include win32lib.ew 
include EInetLib_full.ew 
 
function sslInet(sequence AServer, sequence AUrl, sequence AData, integer blnSSL) 
  sequence aBuffer, s 
  sequence Header 
  sequence sMethod 
  integer BytesRead 
  atom pSession, pConnection, pRequest 
  integer res 
 
  sequence Result 
 
  Result = "" 
 
  pSession = InternetOpen({}) 
  if pSession > 0 then 
    if blnSSL then 
      pConnection = InternetConnect({ 
                      {"session_id" , pSession}, 
                      {"server_name", AServer}, 
                      {"server_port", INTERNET_DEFAULT_HTTPS_PORT}, 
                      {"user_name"  , NULL}, 
                      {"password"   , NULL}, 
                      {"service"    , "HTTP"} 
                    }) 
    else 
      pConnection = InternetConnect({ 
                      {"session_id" , pSession}, 
                      {"server_name", AServer}, 
                      {"server_port", INTERNET_DEFAULT_HTTP_PORT}, 
                      {"user_name"  , NULL}, 
                      {"password"   , NULL}, 
                      {"service"    , "HTTP"} 
                    }) 
    end if 
 
    if pConnection > 0 then 
      if length(AData) then 
        sMethod = "POST" 
      else 
        sMethod = "GET" 
      end if 
       
      if blnSSL then 
        pRequest = HttpOpenRequest({ 
                     {"connection_id", pConnection}, 
                     {"method", sMethod}, 
                     {"object_name", AUrl}, 
                     {"version", NULL}, 
                     {"referrer", NULL}, 
                     {"extra_headers", NULL}, 
                     {"flags", or_bits(INTERNET_FLAG_SECURE, INTERNET_FLAG_KEEP_CONNECTION)} 
                   }) 
      else 
        pRequest = HttpOpenRequest({ 
                     {"connection_id", pConnection}, 
                     {"method", sMethod}, 
                     {"object_name", AUrl}, 
                     {"version", NULL}, 
                     {"referrer", NULL}, 
                     {"extra_headers", NULL}, 
                     {"flags", INTERNET_SERVICE_HTTP} 
                   }) 
      end if 
 
      if pRequest > 0 then 
        Header = "Host: " & AServer & 10 & 
          "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.10) Gecko/2009042316 Firefox/3.0.10" & 10 & 
          "Accept: text/html,application/xhtml&xml,application/xml;q=0.9,*/*;q=0.8" & 10 & 
          "Accept-Language: en-us,en;q=0.5" & 10 & 
          "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7" & 10 & 
          "Keep-Alive: 300" & 10 & 
          "Connection: keep-alive"& 10 & 10 
 
        res = HttpAddRequestHeaders({ 
          {"request_id", pRequest}, 
          {"headers", Header}, 
          {"headers_length", length(Header)}, 
          {"modifiers", HTTP_ADDREQ_FLAG_ADD} 
        }) 
 
        if HttpSendRequest({ 
          {"request_id", pRequest}, 
          {"headers", NULL}, 
          {"headers_length", 0}, 
          {"optional"  , AData}, 
          {"optional_length", length(AData)} 
        }) then 
 
          aBuffer = "" 
          s = InternetReadFile(pRequest, 4096) 
          while length(s) do 
            aBuffer &= s 
            s = InternetReadFile(pRequest, 4096) 
          end while 
        end if 
 
        res = InternetCloseHandle(pRequest) 
      end if 
 
      res = InternetCloseHandle(pConnection) 
    end if 
 
    res = InternetCloseHandle(pSession) 
  else 
    Msg(net_err_msg) 
  end if 
  return aBuffer 
end function 
 
puts(1, sslInet("geneanet.org", "https://gw.geneanet.org/connexion/", 
          "\"_username\": \"myUsername\", \"_password\": \"myPassword\", \"_submit\": \"\"", 1)) 

Added functions seem to work well, but when I try to get a page with InternetReadFile or another similar function, I get an error HTTP 400 The plain HTTP request was sent to HTTPS port. That is supposed to happen when HttpOpenRequest is run without the INTERNET_FLAG_SECURE flag, but I use this flag so I don't understand what's happening.

Jean-Marc